diff --git a/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs b/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs index 937634006..e48c9bfb6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs @@ -15,7 +15,6 @@ using System.Collections.Generic; using System.Diagnostics; -using System.Linq; namespace Microsoft.Python.Analysis.Analyzer { internal static class ActivityTracker { diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index 285fc2f86..0f5915f35 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -39,9 +39,9 @@ internal abstract class AnalysisWalker : PythonWalker { public PythonAst Ast => Eval.Ast; protected ModuleSymbolTable SymbolTable => Eval.SymbolTable; - protected AnalysisWalker(ExpressionEval eval) { + protected AnalysisWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) { Eval = eval; - ImportHandler = new ImportHandler(this); + ImportHandler = new ImportHandler(this, importedVariableHandler); AssignmentHandler = new AssignmentHandler(this); LoopHandler = new LoopHandler(this); ConditionalHandler = new ConditionalHandler(this); diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 1e77bcb22..491d71029 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -71,5 +71,12 @@ public interface IPythonAnalyzer { /// Fires when analysis is complete. /// event EventHandler AnalysisComplete; + + /// + /// Attempts to restore modules analysis from database. + /// + /// + /// + IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index 188933339..02286c469 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -40,9 +40,6 @@ public T GetInScope(string name, IScope scope) where T : class, IMember public void DeclareVariable(string name, IMember value, VariableSource source) => DeclareVariable(name, value, source, default(Location)); - public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module) - => DeclareVariable(name, value, source, new Location(module)); - public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = true) => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index d7c0b3301..65a5a5b3d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -60,7 +60,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { - if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) { + if (node == null || Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library) { return DefaultLocation; } @@ -233,7 +233,8 @@ private IMember GetValueFromName(NameExpression expr, LookupOptions options = Lo } private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOptions = LookupOptions.Normal) { - if (expr?.Target == null || string.IsNullOrEmpty(expr.Name)) { + var memberName = expr?.Name; + if (expr?.Target == null || string.IsNullOrEmpty(memberName)) { return null; } @@ -243,8 +244,9 @@ private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOp } var type = m.GetPythonType(); - var value = type?.GetMember(expr.Name); - type?.AddMemberReference(expr.Name, this, GetLocationOfName(expr)); + var value = type?.GetMember(memberName); + var location = GetLocationOfName(expr); + type?.AddMemberReference(memberName, this, location); if (type is IPythonModule) { return value; diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs index 575d29962..2774ec9e7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Analyzer.Symbols; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing.Ast; @@ -28,7 +29,7 @@ internal sealed class FunctionCallEvaluator: AnalysisWalker { private readonly FunctionDefinition _function; private IMember _result; - public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval) { + public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval, SimpleImportedVariableHandler.Instance) { _declaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule)); _eval = eval ?? throw new ArgumentNullException(nameof(eval)); _function = fd ?? throw new ArgumentNullException(nameof(fd)); diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index 4d09199d1..f57244f9e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -59,12 +59,21 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor for (var i = 0; i < names.Count; i++) { var memberName = names[i].Name; - if (!string.IsNullOrEmpty(memberName)) { - var nameExpression = asNames[i] ?? names[i]; - var variableName = nameExpression?.Name ?? memberName; - if (!string.IsNullOrEmpty(variableName)) { - DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression); - } + if (string.IsNullOrEmpty(memberName)) { + continue; + } + + var nameExpression = asNames[i] ?? names[i]; + var variableName = nameExpression?.Name ?? memberName; + if (!string.IsNullOrEmpty(variableName)) { + DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression); + } + + if (imports is IImportChildrenSource cs + && cs.TryGetChildImport(memberName, out var csr) + && HandleImportSearchResult(csr, variableModule, null, names[i], out var childModule)) { + + _importedVariableHandler.EnsureModule(childModule); } } } @@ -74,10 +83,11 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, IImport // from self import * won't define any new members return; } + // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore. var memberNames = imports is ImplicitPackageImport ? variableModule.GetMemberNames() - : variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); + : _importedVariableHandler.GetMemberNames(variableModule).ToArray(); foreach (var memberName in memberNames) { DeclareVariable(variableModule, memberName, imports, memberName, importPosition, nameExpression); @@ -104,23 +114,36 @@ private void DeclareVariable(PythonVariableModule variableModule, string memberN value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); // First try exported or child submodules. - value = value ?? variableModule.GetMember(memberName); + var member = variableModule.GetMember(memberName); // Value may be variable or submodule. If it is variable, we need it in order to add reference. - var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; - value = variable?.Value?.Equals(value) == true ? variable : value; - - // If nothing is exported, variables are still accessible. - value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType; + var variable = _importedVariableHandler.GetVariable(variableModule, memberName); + + if (member is PythonVariableModule vm && vm.Equals(variable?.Value)) { + // If member is submodule, use actual variable so it can be linked through for goto definition. + value = variable; + } else if (value == null) { + if (member is PythonVariableModule) { + // If member is submodule, use it. + value = member; + } else if (variable?.Value != null) { + // Otherwise use variable, if available so references can be linked. + value = variable; + } else if (member != null) { + value = member; + } else { + value = Eval.UnknownType; + } + } } - + // Do not allow imported variables to override local declarations var canOverwrite = CanOverwriteVariable(variableName, importPosition, value); - + // Do not declare references to '*' var locationExpression = nameLocation is NameExpression nex && nex.Name == "*" ? null : nameLocation; Eval.DeclareVariable(variableName, value, VariableSource.Import, locationExpression, canOverwrite); - + // Make sure module is loaded and analyzed. if (value is IPythonModule m) { ModuleResolution.GetOrLoadModule(m.Name); @@ -132,8 +155,8 @@ private bool CanOverwriteVariable(string name, int importPosition, IMember newVa if (v == null) { return true; // Variable does not exist } - - if(newValue.IsUnknown()) { + + if (newValue.IsUnknown()) { return false; // Do not overwrite potentially good value with unknowns. } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs new file mode 100644 index 000000000..b269186d6 --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs @@ -0,0 +1,26 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Values; + +namespace Microsoft.Python.Analysis.Analyzer.Handlers { + internal interface IImportedVariableHandler { + IEnumerable GetMemberNames(PythonVariableModule variableModule); + IVariable GetVariable(in PythonVariableModule module, in string name); + void EnsureModule(in PythonVariableModule module); + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 62170c8be..66e0cdc0b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -29,9 +29,12 @@ namespace Microsoft.Python.Analysis.Analyzer.Handlers { internal sealed partial class ImportHandler : StatementHandler { + private readonly IImportedVariableHandler _importedVariableHandler; private readonly Dictionary _variableModules = new Dictionary(); - public ImportHandler(AnalysisWalker walker) : base(walker) { } + public ImportHandler(AnalysisWalker walker, in IImportedVariableHandler importedVariableHandler) : base(walker) { + _importedVariableHandler = importedVariableHandler; + } public bool HandleImport(ImportStatement node) { if (Module.ModuleType == ModuleType.Specialized) { @@ -65,7 +68,9 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa lastModule = default; break; } + resolvedModules[i] = (nameExpression.Name, lastModule); + _importedVariableHandler.EnsureModule(lastModule); } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') @@ -89,15 +94,6 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]); } } - - // import a.b.c.d => declares a, b in the current module, c in b, d in c. - for (var i = 1; i < resolvedModules.Length - 1; i++) { - var (childName, childModule) = resolvedModules[i + 1]; - if (!string.IsNullOrEmpty(childName) && childModule != null) { - var parent = resolvedModules[i].module; - parent?.AddChildModule(childName, childModule); - } - } } private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in Node location, out PythonVariableModule variableModule) { @@ -112,8 +108,7 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV return TryGetPackageFromImport(packageImport, parent, out variableModule); case RelativeImportBeyondTopLevel importBeyondTopLevel: var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName); - Eval.ReportDiagnostics(Eval.Module.Uri, - new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); + Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); variableModule = default; return false; case ImportNotFound importNotFound: @@ -177,7 +172,7 @@ private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImpor return false; } } - + return true; } @@ -196,24 +191,22 @@ private void MakeUnresolvedImport(string variableName, string moduleName, Node l } private PythonVariableModule GetOrCreateVariableModule(in string fullName, in PythonVariableModule parentModule, in string memberName) { - if (_variableModules.TryGetValue(fullName, out var variableModule)) { - return variableModule; + if (!_variableModules.TryGetValue(fullName, out var variableModule)) { + variableModule = new PythonVariableModule(fullName, Eval.Interpreter); + _variableModules[fullName] = variableModule; } - - variableModule = new PythonVariableModule(fullName, Eval.Interpreter); - _variableModules[fullName] = variableModule; + parentModule?.AddChildModule(memberName, variableModule); return variableModule; } private PythonVariableModule GetOrCreateVariableModule(in IPythonModule module, in PythonVariableModule parentModule, in string memberName) { var moduleFullName = module.Name; - if (_variableModules.TryGetValue(moduleFullName, out var variableModule)) { - return variableModule; + if (!_variableModules.TryGetValue(moduleFullName, out var variableModule)) { + variableModule = new PythonVariableModule(module); + _variableModules[moduleFullName] = variableModule; } - variableModule = new PythonVariableModule(module); - _variableModules[moduleFullName] = variableModule; parentModule?.AddChildModule(memberName, variableModule); return variableModule; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs new file mode 100644 index 000000000..28c4f4c5f --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs @@ -0,0 +1,130 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer.Handlers { + internal sealed class LoopImportedVariableHandler : IImportedVariableHandler { + private readonly Dictionary _walkers = new Dictionary(); + private readonly IReadOnlyDictionary _asts; + private readonly IReadOnlyDictionary _cachedVariables; + private readonly IServiceContainer _services; + private readonly Func _isCanceled; + + public IReadOnlyCollection Walkers => _walkers.Values; + + public LoopImportedVariableHandler(in IServiceContainer services, + in IReadOnlyDictionary asts, + in IReadOnlyDictionary cachedVariables, + in Func isCanceled) { + + _services = services; + _isCanceled = isCanceled; + _asts = asts; + _cachedVariables = cachedVariables; + } + + public IEnumerable GetMemberNames(PythonVariableModule variableModule) { + var module = variableModule.Module; + if (module == null || _isCanceled()) { + return Enumerable.Empty(); + } + + var key = new AnalysisModuleKey(module); + if (_walkers.TryGetValue(key, out var walker)) { + return GetMemberNames(walker, variableModule); + } + + if (!_asts.TryGetValue(key, out var ast)) { + return _cachedVariables.TryGetValue(key, out var variables) + ? variables.Names + : variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); + } + + walker = WalkModule(module, ast); + return walker != null ? GetMemberNames(walker, variableModule) : module.GetMemberNames(); + } + + public IVariable GetVariable(in PythonVariableModule variableModule, in string name) { + var module = variableModule.Module; + if (module == null || _isCanceled()) { + return default; + } + + var key = new AnalysisModuleKey(module); + if (_walkers.TryGetValue(key, out var walker)) { + return walker.Eval.GlobalScope?.Variables[name]; + } + + if (!_asts.TryGetValue(key, out var ast)) { + return _cachedVariables.TryGetValue(key, out var variables) ? variables[name] : default; + } + + _walkers[key] = walker = WalkModule(module, ast); + var gs = walker != null ? walker.Eval.GlobalScope : module.GlobalScope; + return gs?.Variables[name]; + } + + public void EnsureModule(in PythonVariableModule variableModule) { + var module = variableModule.Module; + if (module == null || _isCanceled()) { + return; + } + EnsureModule(module); + } + + public void EnsureModule(in IPythonModule module) { + if (module == null || _isCanceled()) { + return; + } + var key = new AnalysisModuleKey(module); + if (!_walkers.ContainsKey(key) && _asts.TryGetValue(key, out var ast)) { + WalkModule(module, ast); + } + } + + public ModuleWalker WalkModule(IPythonModule module, PythonAst ast) { + var analyzer = _services.GetService(); + var analysis = analyzer.TryRestoreCachedAnalysis(module); + if (analysis != null) { + return null; + } + + // If module has stub, make sure it is processed too. + if (module.Stub?.Analysis is EmptyAnalysis) { + WalkModule(module.Stub, module.GetAst()); + } + + var eval = new ExpressionEval(_services, module, ast); + var walker = new ModuleWalker(eval, this); + + _walkers[new AnalysisModuleKey(module)] = walker; + ast.Walk(walker); + walker.Complete(); + return walker; + } + + private static IEnumerable GetMemberNames(ModuleWalker walker, PythonVariableModule variableModule) + => walker.StarImportMemberNames ?? walker.GlobalScope.GetExportableVariableNames().Concat(variableModule.ChildrenNames).Distinct().Where(s => !s.StartsWithOrdinal("_")); + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/SimpleImportedVariableHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/SimpleImportedVariableHandler.cs new file mode 100644 index 000000000..86d08832e --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/SimpleImportedVariableHandler.cs @@ -0,0 +1,36 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Analyzer.Handlers { + internal sealed class SimpleImportedVariableHandler : IImportedVariableHandler { + public static IImportedVariableHandler Instance { get; } = new SimpleImportedVariableHandler(); + + private SimpleImportedVariableHandler() {} + + public IEnumerable GetMemberNames(PythonVariableModule variableModule) + => variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); + + public IVariable GetVariable(in PythonVariableModule module, in string memberName) + => module.Analysis?.GlobalScope?.Variables[memberName]; + + public void EnsureModule(in PythonVariableModule module) { } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 68f7a1cb1..201f7660c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -18,6 +18,7 @@ using System.Linq; using System.Threading; using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; @@ -37,8 +38,7 @@ internal class ModuleWalker : AnalysisWalker { private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast, CancellationToken cancellationToken) - : base(new ExpressionEval(services, module, ast)) { + public ModuleWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) : base(eval, importedVariableHandler) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; _cancellationToken = CancellationToken.None; } @@ -206,7 +206,7 @@ public void Complete() { new StubMerger(Eval).MergeStub(_stubAnalysis, _cancellationToken); if (_allIsUsable && _allReferencesCount >= 1 && GlobalScope.Variables.TryGetVariable(AllVariableName, out var variable) - && variable?.Value is IPythonCollection collection && collection.IsExact) { + && variable.Value is IPythonCollection collection && collection.IsExact) { StarImportMemberNames = collection.Contents .OfType() .Select(c => c.GetString()) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 67e6a249d..4a2756f20 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -20,6 +20,8 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; @@ -65,6 +67,7 @@ public void Dispose() { _disposeToken.TryMarkDisposed(); } + #region IPythonAnalyzer public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) => _analysisCompleteEvent.WaitAsync(cancellationToken); @@ -206,6 +209,26 @@ public IReadOnlyList LoadedModules { public event EventHandler AnalysisComplete; + public IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module) { + var moduleType = module.ModuleType; + var moduleDatabaseService = _services.GetService(); + if (!moduleType.CanBeCached() || moduleDatabaseService == null || !moduleDatabaseService.ModuleExistsInStorage(module.Name, module.FilePath)) { + return null; + } + + if (moduleDatabaseService.TryRestoreGlobalScope(module, out var gs)) { + _log?.Log(TraceEventType.Verbose, "Restored from database: ", module.Name); + var analysis = new DocumentAnalysis((IDocument)module, 1, gs, new ExpressionEval(_services, module, module.GetAst()), Array.Empty()); + gs.ReconstructVariables(); + return analysis; + } + + _log?.Log(TraceEventType.Verbose, "Restore from database failed for module ", module.Name); + + return null; + } + #endregion + internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) { _analysisCompleteEvent.Set(); AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index d063eb9a6..caeb42dca 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime; @@ -21,14 +22,19 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Caching; +using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; +using Microsoft.Python.Core.OS; +using Microsoft.Python.Core.Services; using Microsoft.Python.Core.Testing; using Microsoft.Python.Parsing.Ast; @@ -43,11 +49,14 @@ internal sealed class PythonAnalyzerSession { private readonly CancellationToken _analyzerCancellationToken; private readonly IServiceContainer _services; private readonly IDiagnosticsService _diagnosticsService; + private readonly IOSPlatform _platformService; private readonly IProgressReporter _progress; private readonly IPythonAnalyzer _analyzer; private readonly ILogger _log; private readonly bool _forceGC; private readonly IModuleDatabaseService _moduleDatabaseService; + private readonly PathResolverSnapshot _modulesPathResolver; + private readonly PathResolverSnapshot _typeshedPathResolver; private State _state; private bool _isCanceled; @@ -75,6 +84,7 @@ public PythonAnalyzerSession(IServiceContainer services, _services = services; _startNextSession = startNextSession; + _analyzerCancellationToken = analyzerCancellationToken; Version = version; AffectedEntriesCount = walker?.AffectedValues.Count ?? 1; @@ -84,10 +94,15 @@ public PythonAnalyzerSession(IServiceContainer services, _forceGC = forceGC; _diagnosticsService = _services.GetService(); + _platformService = _services.GetService(); _analyzer = _services.GetService(); _log = _services.GetService(); _moduleDatabaseService = _services.GetService(); _progress = progress; + + var interpreter = _services.GetService(); + _modulesPathResolver = interpreter.ModuleResolution.CurrentPathResolver; + _typeshedPathResolver = interpreter.TypeshedResolution.CurrentPathResolver; } public void Start(bool analyzeEntry) { @@ -151,6 +166,10 @@ private async Task StartAsync() { totalMilliseconds = Math.Round(totalMilliseconds, 2); (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); + //#if DEBUG + // var notReady = _analyzer.LoadedModules.Where(m => (m.ModuleType == ModuleType.Library || m.ModuleType == ModuleType.Stub) && m.Analysis is EmptyAnalysis).ToArray(); + // Debug.Assert(notReady.Length == 0); + //#endif } } } @@ -185,24 +204,30 @@ private static void LogResults(ILogger logger, double elapsed, int originalRemai } private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { - IDependencyChainNode node; + IDependencyChainNode node; var remaining = 0; var ace = new AsyncCountdownEvent(0); - bool isCanceled; while ((node = await _walker.GetNextAsync(_analyzerCancellationToken)) != null) { + bool isCanceled; lock (_syncObj) { isCanceled = _isCanceled; } - if (isCanceled && !node.Value.NotAnalyzed) { - remaining++; - node.MoveNext(); - continue; + if (isCanceled) { + switch (node) { + case IDependencyChainLoopNode loop: + // Loop analysis is not cancellable or else small + // inner loops of a larger loop will not be analyzed + // correctly as large loop may cancel inner loop pass. + break; + case IDependencyChainSingleNode single when !single.Value.NotAnalyzed: + remaining++; + node.MoveNext(); + continue; + } } - ActivityTracker.OnEnqueueModule(node.Value.Module.FilePath); - var taskLimitReached = false; lock (_syncObj) { _runningTasks++; @@ -230,135 +255,286 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { return remaining; } - - private bool IsAnalyzedLibraryInLoop(IDependencyChainNode node, IDocumentAnalysis currentAnalysis) - => !node.HasMissingDependencies && currentAnalysis is LibraryAnalysis && node.IsWalkedWithDependencies && node.IsValidVersion; - - private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) + private void RunAnalysis(IDependencyChainNode node, Stopwatch stopWatch) => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, null, stopWatch), null); - private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) + private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) => Task.Run(() => Analyze(node, ace, stopWatch)); - /// - /// Performs analysis of the document. Returns document global scope - /// with declared variables and inner scopes. Does not analyze chain - /// of dependencies, it is intended for the single file analysis. - /// - private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) { + private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) { + var loopAnalysis = false; try { - var entry = node.Value; - if (!CanUpdateAnalysis(entry, node, _walker.Version, out var module, out var ast, out var currentAnalysis)) { - return; + switch (node) { + case IDependencyChainSingleNode single: + try { + Analyze(single, stopWatch); + } catch (OperationCanceledException oce) { + single.Value.TryCancel(oce, _walker.Version); + LogCanceled(single.Value.Module); + } catch (Exception exception) { + single.Value.TrySetException(exception, _walker.Version); + node.MarkWalked(); + LogException(single.Value, exception); + } + + break; + case IDependencyChainLoopNode loop: + try { + loopAnalysis = true; + AnalyzeLoop(loop, stopWatch); + } catch (OperationCanceledException) { + //loop.Value.TryCancel(oce, _walker.Version); + //LogCanceled(single.Value.Module); + } catch (Exception exception) { + //loop.Value.TrySetException(exception, _walker.Version); + node.MarkWalked(); + LogException(loop, exception); + } + + break; } - var startTime = stopWatch.Elapsed; - AnalyzeEntry(node, entry, module, ast, _walker.Version); - - LogCompleted(node, module, stopWatch, startTime); - } catch (OperationCanceledException oce) { - node.Value.TryCancel(oce, _walker.Version); - LogCanceled(node.Value.Module); - } catch (Exception exception) { - node.Value.TrySetException(exception, _walker.Version); - node.MarkWalked(); - LogException(node.Value.Module, exception); } finally { node.MoveNext(); + bool isCanceled; lock (_syncObj) { - if (!_isCanceled) { - _progress.ReportRemaining(_walker.Remaining); - } - _runningTasks--; - ace?.Signal(); + isCanceled = _isCanceled; + } + + if (!isCanceled || loopAnalysis) { + _progress.ReportRemaining(_walker.Remaining); } + + Interlocked.Decrement(ref _runningTasks); + ace?.Signal(); + } + } + + /// + /// Performs analysis of the document. Returns document global scope + /// with declared variables and inner scopes. Does not analyze chain + /// of dependencies, it is intended for the single file analysis. + /// + private void Analyze(IDependencyChainSingleNode node, Stopwatch stopWatch) { + ActivityTracker.OnEnqueueModule(node.Value.Module.FilePath); + var entry = node.Value; + if (!CanUpdateAnalysis(entry, _walker.Version, out var module, out var ast)) { + return; } + var startTime = stopWatch.Elapsed; + AnalyzeEntry(node, entry, module, ast, _walker.Version); + + LogCompleted(node, module, stopWatch, startTime); } private void AnalyzeEntry() { var stopWatch = _log != null ? Stopwatch.StartNew() : null; try { - if (!CanUpdateAnalysis(_entry, null, Version, out var module, out var ast, out var currentAnalysis)) { + if (!CanUpdateAnalysis(_entry, Version, out var module, out var ast)) { return; } var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; AnalyzeEntry(null, _entry, module, ast, Version); - LogCompleted(null, module, stopWatch, startTime); + LogCompleted(module, stopWatch, startTime); } catch (OperationCanceledException oce) { _entry.TryCancel(oce, Version); LogCanceled(_entry.Module); } catch (Exception exception) { _entry.TrySetException(exception, Version); - LogException(_entry.Module, exception); + LogException(_entry, exception); } finally { stopWatch?.Stop(); } } - private bool CanUpdateAnalysis( - PythonAnalyzerEntry entry, - IDependencyChainNode node, - int version, - out IPythonModule module, - out PythonAst ast, - out IDocumentAnalysis currentAnalysis) { + private void AnalyzeLoop(IDependencyChainLoopNode loopNode, Stopwatch stopWatch) { + var version = _walker.Version; + var entries = new Dictionary(); + var variables = new Dictionary<(AnalysisModuleKey Module, string Name), int>(); + var importNames = new List<(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, string ToName)>(); + var cachedVariables = new Dictionary(); + var asts = new Dictionary(); + var startTime = stopWatch.Elapsed; + + // Note: loop analysis is not cancellable. The reason is that when smaller loop + // appears inside a larger loop gets canceled, it will not be re-walked during + // the outer loop analysis. For example, functools <=> _functools loop and + // the related CircularDependencyFunctools test. + foreach (var entry in loopNode.Values) { + ActivityTracker.OnEnqueueModule(entry.Module.FilePath); + if (!CanUpdateAnalysis(entry, Version, out var module, out var ast)) { + _log?.Log(TraceEventType.Verbose, $"Analysis of loop canceled."); + return; + } - if (!entry.CanUpdateAnalysis(version, out module, out ast, out currentAnalysis)) { - if (IsAnalyzedLibraryInLoop(node, currentAnalysis)) { - // Library analysis exists, don't analyze again - return false; + var moduleKey = new AnalysisModuleKey(module); + entries[moduleKey] = (module, entry); + var analysis = _analyzer.TryRestoreCachedAnalysis(module); + if (analysis != null) { + AddLoopImportsFromCachedAnalysis(importNames, variables, moduleKey, analysis); + cachedVariables.Add(moduleKey, analysis.GlobalScope.Variables); + } else { + AddLoopImportsFromAst(importNames, variables, moduleKey, ast); + asts.Add(moduleKey, ast); } - if (ast == null) { - if (currentAnalysis == null) { - // Entry doesn't have ast yet. There should be at least one more session. - Cancel(); - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled (no AST yet)."); - return false; - } - //Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); - return false; + } + + if (asts.Count == 0) { + // Fully cached loop + if (_log != null && _log.LogLevel == TraceEventType.Verbose) { + var names = string.Join(", ", cachedVariables.Select(v => v.Key.Name)); + _log?.Log(TraceEventType.Verbose, $"Fully cached modules cycle: {names}"); + } + return; + } + + var imports = new List<(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, int ToPosition)>(); + foreach (var (fromModule, fromPosition, toModule, toName) in importNames) { + if (!entries.ContainsKey(toModule)) { + continue; } - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled. Version: {version}, current: {module.Analysis.Version}."); + if (toName == null) { + imports.Add((fromModule, fromPosition, toModule, 0)); + } else if (variables.TryGetValue((toModule, toName), out var toPosition)) { + imports.Add((fromModule, fromPosition, toModule, toPosition)); + } + } + + var startingKeys = LocationLoopResolver.FindStartingItems(imports); + var variableHandler = new LoopImportedVariableHandler(_services, asts, cachedVariables, () => false); + foreach (var key in startingKeys) { + if (asts.TryGetValue(key, out var startingAst) && entries.TryGetValue(key, out var me)) { + variableHandler.WalkModule(me.Module, startingAst); + } + } + + foreach (var walker in variableHandler.Walkers) { + asts.Remove(new AnalysisModuleKey(walker.Module)); + } + + while (asts.Count > 0) { + var (moduleKey, ast) = asts.First(); + variableHandler.WalkModule(entries[moduleKey].Module, ast); + + foreach (var walker in variableHandler.Walkers) { + asts.Remove(new AnalysisModuleKey(walker.Module)); + } + } + + foreach (var walker in variableHandler.Walkers) { + var module = (IDocument)walker.Module; + var moduleKey = new AnalysisModuleKey(module); + if (entries.TryGetValue(moduleKey, out var e)) { + var analysis = CreateAnalysis(null, module, walker.Ast, version, walker); + CompleteAnalysis(e.Entry, module, version, analysis); + } + } + + loopNode.MarkWalked(); + LogCompleted(loopNode, entries.Values.Select(v => v.Module), stopWatch, startTime); + } + + private void AddLoopImportsFromCachedAnalysis(in List<(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, string ToName)> unresolvedImports, + in Dictionary<(AnalysisModuleKey Module, string Name), int> variables, + in AnalysisModuleKey moduleKey, + in IDocumentAnalysis analysis) { + + foreach (var variable in analysis.GlobalScope.Variables) { + var key = (moduleKey, variable.Name); + var location = variable.Location.IndexSpan.Start; + if (!variables.TryGetValue(key, out var currentLocation) || currentLocation > location) { + variables[key] = location; + } + } + } + + private void AddLoopImportsFromAst( + in List<(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, string ToName)> imports, + in Dictionary<(AnalysisModuleKey Module, string Name), int> variables, + in AnalysisModuleKey moduleKey, + in PythonAst ast) { + + var pathResolver = moduleKey.IsTypeshed ? _typeshedPathResolver : _modulesPathResolver; + var walker = new ImportExportWalker(ast, _platformService, pathResolver, moduleKey.FilePath, moduleKey.IsTypeshed); + walker.Walk(); + + foreach (var export in walker.Exports) { + var key = (moduleKey, export.Name); + var location = export.Location.Start; + if (!variables.TryGetValue(key, out var currentLocation) || currentLocation > location) { + variables[key] = location; + } + } + + foreach (var (toModule, name, location) in walker.Imports) { + imports.Add((moduleKey, location.Start, toModule, name)); + } + } + + private bool CanUpdateAnalysis(PythonAnalyzerEntry entry, int version, out IPythonModule module, out PythonAst ast) { + if (entry.CanUpdateAnalysis(version, out module, out ast, out var currentAnalysis)) { + return true; + } + + if (ast == null) { + if (currentAnalysis == null) { + // Entry doesn't have ast yet. There should be at least one more session. + Cancel(); + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled (no AST yet)."); + return false; + } + //Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); return false; } - return true; + + _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled. Version: {version}, current: {module.Analysis.Version}."); + return false; } - private void AnalyzeEntry(IDependencyChainNode node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { + private void AnalyzeEntry(IDependencyChainSingleNode node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { // Now run the analysis. var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); Debug.Assert(ast != null); - var analysis = DoAnalyzeEntry(node, module, ast, version); + var analysis = RestoreOrAnalyzeModule(node, module, ast, version); _analyzerCancellationToken.ThrowIfCancellationRequested(); if (analysis != null) { - analyzable?.NotifyAnalysisComplete(analysis); - entry.TrySetAnalysis(analysis, version); + CompleteAnalysis(entry, module, version, analysis); + } + } - if (module.ModuleType == ModuleType.User) { - var linterDiagnostics = _analyzer.LintModule(module); - _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); - } + private void CompleteAnalysis(PythonAnalyzerEntry entry, IPythonModule module, int version, IDocumentAnalysis analysis) { + var analyzable = module as IAnalyzable; + analyzable?.NotifyAnalysisComplete(analysis); + entry.TrySetAnalysis(analysis, version); + + if (module.ModuleType != ModuleType.User) { + return; } + + var linterDiagnostics = _analyzer.LintModule(module); + _diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter); } - private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode node, IPythonModule module, PythonAst ast, int version) { - var analysis = TryRestoreCachedAnalysis(node, module); + private IDocumentAnalysis RestoreOrAnalyzeModule(IDependencyChainSingleNode node, IPythonModule module, PythonAst ast, int version) { + var analysis = _analyzer.TryRestoreCachedAnalysis(module); if (analysis != null) { + MarkNodeWalked(node); return analysis; } - var walker = new ModuleWalker(_services, module, ast, _analyzerCancellationToken); + var eval = new ExpressionEval(_services, module, ast); + var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance); ast.Walk(walker); walker.Complete(); return CreateAnalysis(node, (IDocument)module, ast, version, walker); } - private bool MarkNodeWalked(IDependencyChainNode node) { + private bool MarkNodeWalked(IDependencyChainNode node) { bool isCanceled; lock (_syncObj) { isCanceled = _isCanceled; @@ -369,27 +545,7 @@ private bool MarkNodeWalked(IDependencyChainNode node) { return isCanceled; } - private IDocumentAnalysis TryRestoreCachedAnalysis(IDependencyChainNode node, IPythonModule module) { - var moduleType = module.ModuleType; - if (moduleType.CanBeCached() && _moduleDatabaseService?.ModuleExistsInStorage(module.Name, module.FilePath) == true) { - if (_moduleDatabaseService.TryRestoreGlobalScope(module, out var gs)) { - if (_log != null) { - _log.Log(TraceEventType.Verbose, "Restored from database: ", module.Name); - } - var analysis = new DocumentAnalysis((IDocument)module, 1, gs, new ExpressionEval(_services, module, module.GetAst()), Array.Empty()); - gs.ReconstructVariables(); - MarkNodeWalked(node); - return analysis; - } else { - if (_log != null) { - _log.Log(TraceEventType.Verbose, "Restore from database failed for module ", module.Name); - } - } - } - return null; - } - - private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker) { + private IDocumentAnalysis CreateAnalysis(IDependencyChainSingleNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker) { var canHaveLibraryAnalysis = false; // Don't try to drop builtins; it causes issues elsewhere. @@ -404,18 +560,20 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode x is ImportStatement || x is FromImportStatement); + ast.ReduceToImports(); document.SetAst(ast); var eval = new ExpressionEval(walker.Eval.Services, document, ast); @@ -427,14 +585,31 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IEnumerable modules, Stopwatch stopWatch, TimeSpan startTime) { + if (_log != null) { + var moduleNames = modules.Select(m => "{0}({1})".FormatInvariant(m.Name, m.Analysis is LibraryAnalysis ? "Library" : m.ModuleType.ToString())); + var elapsed = Math.Round((stopWatch.Elapsed - startTime).TotalMilliseconds, 2); + var message = $"Analysis of modules loop on depth {node.VertexDepth} in {elapsed} ms:"; + _log.Log(TraceEventType.Verbose, message); + foreach (var name in moduleNames) { + _log.Log(TraceEventType.Verbose, $" {name}"); + } + } + } - private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { + private void LogCompleted(IDependencyChainSingleNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { if (_log != null) { - var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; + var completed = module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; var elapsed = Math.Round((stopWatch.Elapsed - startTime).TotalMilliseconds, 2); - var message = node != null - ? $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {elapsed} ms." - : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {elapsed} ms."; + var message = $"Analysis of {module.Name} ({module.ModuleType}) on depth {node.VertexDepth} {completed} in {elapsed} ms."; + _log.Log(TraceEventType.Verbose, message); + } + } + + private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { + if (_log != null) { + var elapsed = Math.Round((stopWatch.Elapsed - startTime).TotalMilliseconds, 2); + var message = $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {elapsed} ms."; _log.Log(TraceEventType.Verbose, message); } } @@ -445,9 +620,21 @@ private void LogCanceled(IPythonModule module) { } } - private void LogException(IPythonModule module, Exception exception) { + private void LogException(PythonAnalyzerEntry entry, Exception exception) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) failed. {exception}"); + } + + if (TestEnvironment.Current != null) { + ExceptionDispatchInfo.Capture(exception).Throw(); + } + } + + private void LogException(IDependencyChainLoopNode node, Exception exception) { if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. {exception}"); + var moduleNames = string.Join(", ", node.Values.Select(e => $"{e.Module.Name}({e.Module.ModuleType})")); + var message = $"Analysis of modules loop [{moduleNames}] failed. {exception}"; + _log.Log(TraceEventType.Verbose, message); } if (TestEnvironment.Current != null) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 5b29b1194..12b03e165 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -81,29 +81,45 @@ public override void Evaluate() { Result = _function; } - private IPythonType TryDetermineReturnValue() { - var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation, LookupOptions.All); - if (!annotationType.IsUnknown()) { - // Annotations are typically types while actually functions return - // instances unless specifically annotated to a type such as Type[T]. - // TODO: try constructing argument set from types. Consider Tuple[_T1, _T2] where _T1 = TypeVar('_T1', str, bytes) - var t = annotationType.CreateInstance(ArgumentSet.Empty(FunctionDefinition.ReturnAnnotation, Eval)); - // If instance could not be created, such as when return type is List[T] and - // type of T is not yet known, just use the type. - var instance = t.IsUnknown() ? (IMember)annotationType : t; - _overload.SetReturnValue(instance, true); _overload.SetReturnValue(instance, true); - } else { - // Check if function is a generator - var suite = FunctionDefinition.Body as SuiteStatement; - var yieldExpr = suite?.Statements.OfType().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault(); - if (yieldExpr != null) { - // Function return is an iterator - var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType; - var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue); - _overload.SetReturnValue(returnValue, true); - } + public static IMember GetReturnValueFromAnnotation(ExpressionEval eval, Expression annotation) { + if (eval == null || annotation == null) { + return null; + } + + var annotationType = eval.GetTypeFromAnnotation(annotation, LookupOptions.All); + if (annotationType.IsUnknown()) { + return null; + } + + // Annotations are typically types while actually functions return + // instances unless specifically annotated to a type such as Type[T]. + // TODO: try constructing argument set from types. Consider Tuple[_T1, _T2] where _T1 = TypeVar('_T1', str, bytes) + var t = annotationType.CreateInstance(ArgumentSet.Empty(annotation, eval)); + // If instance could not be created, such as when return type is List[T] and + // type of T is not yet known, just use the type. + var instance = t.IsUnknown() ? (IMember)annotationType : t; + return instance; + } + + private IMember TryDetermineReturnValue() { + var returnType = GetReturnValueFromAnnotation(Eval, FunctionDefinition.ReturnAnnotation); + if (returnType != null) { + _overload.SetReturnValue(returnType, true); + return returnType; } - return annotationType; + + // Check if function is a generator + var suite = FunctionDefinition.Body as SuiteStatement; + var yieldExpr = suite?.Statements.OfType().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault(); + if (yieldExpr != null) { + // Function return is an iterator + var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType; + var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue); + _overload.SetReturnValue(returnValue, true); + return returnValue; + } + + return null; } private void CheckValidOverload(IReadOnlyList parameters) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs index b2f63639c..6cc3c00f3 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/MemberEvaluator.cs @@ -16,13 +16,14 @@ using System; using System.Diagnostics; using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Symbols { [DebuggerDisplay("{Target.Name}")] internal abstract class MemberEvaluator : AnalysisWalker { - protected MemberEvaluator(ExpressionEval eval, ScopeStatement target) : base(eval) { + protected MemberEvaluator(ExpressionEval eval, ScopeStatement target) : base(eval, SimpleImportedVariableHandler.Instance) { Target = target ?? throw new ArgumentNullException(nameof(target)); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index 07c1afb18..44b7fcf1f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -112,7 +112,6 @@ private PythonClassType CreateClass(ClassDefinition cd) { return cls; } - private void AddFunctionOrProperty(FunctionDefinition fd) { var declaringType = fd.Parent != null && _typeMap.TryGetValue(fd.Parent, out var t) ? t : null; if (!TryAddProperty(fd, declaringType)) { @@ -166,7 +165,7 @@ private PythonFunctionOverload GetOverloadFromStub(FunctionDefinition node) { if (t is IPythonFunctionType f) { return f.Overloads .OfType() - .FirstOrDefault(o => o.Parameters.Count == node.Parameters.Where(p => !p.IsPositionalOnlyMarker).Count()); + .FirstOrDefault(o => o.Parameters.Count == node.Parameters.Count(p => !p.IsPositionalOnlyMarker)); } return null; } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs index a0c3e4cca..381fc40f9 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyCollector.cs @@ -29,9 +29,9 @@ internal sealed class DependencyCollector { public ISet Dependencies { get; } = new HashSet(); - public DependencyCollector(IPythonModule module, bool? isTypeShed = null) { + public DependencyCollector(IPythonModule module, bool? isTypeshed = null) { _module = module; - _isTypeshed = isTypeShed ?? module.IsTypeshed; + _isTypeshed = isTypeshed ?? module.IsTypeshed; _moduleResolution = module.Interpreter.ModuleResolution; _pathResolver = _isTypeshed ? module.Interpreter.TypeshedResolution.CurrentPathResolver diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 747adf52b..faf05d656 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -13,7 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -207,7 +209,7 @@ public bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDepend return false; } - if (!TryResolveLoops(walkingGraph, loopsCount, version, out var totalNodesCount)) { + if (!TryResolveLoops(walkingGraph, loopsCount, version, out var loopNodes)) { walker = default; return false; } @@ -218,14 +220,15 @@ public bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDepend return false; } + var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value); + + walkingGraph = walkingGraph.AddRange(loopNodes); foreach (var vertex in walkingGraph) { vertex.Seal(); - vertex.SecondPass?.Seal(); } - var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value); var startingVertices = walkingGraph.Where(v => !v.HasIncoming); - walker = new DependencyChainWalker(this, startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version); + walker = new DependencyChainWalker(this, startingVertices, affectedValues, depths, missingKeys, version); return version == _version; } @@ -243,11 +246,7 @@ private bool TryBuildReverseGraph(in ImmutableArray[vertices.Count]; - foreach (var vertex in vertices) { - if (vertex == null) { - continue; - } - + foreach (var vertex in vertices.Where(vertex => vertex != null)) { if (version != _version) { return false; } @@ -270,10 +269,8 @@ private bool TryBuildReverseGraph(in ImmutableArray vertex != null && !vertex.IsSealed)) { + vertex.Seal(outgoingVertices[vertex.Index]); } return true; @@ -434,109 +431,62 @@ private static bool SetLoopNumber(WalkingVertex vertex, Stack> graph, int loopsCount, int version, out int totalNodesCount) { + private bool TryResolveLoops(in ImmutableArray> graph, int loopsCount, int version, out ImmutableArray> loopVertices) { + loopVertices = ImmutableArray>.Empty; if (loopsCount == 0) { - totalNodesCount = graph.Count; return true; } - // Create vertices for second pass - var inLoopsCount = 0; - var secondPassLoops = new List>[loopsCount]; + // Create independent walking vertices for vertex loops + for (var i = 0; i < loopsCount; i++) { + loopVertices = loopVertices.Add(new WalkingVertex(i)); + } + + // Break internal loop connections foreach (var vertex in graph) { if (vertex.IsInLoop) { - var secondPassVertex = vertex.CreateSecondPassVertex(); var loopNumber = vertex.LoopNumber; - if (secondPassLoops[loopNumber] == null) { - secondPassLoops[loopNumber] = new List> { secondPassVertex }; - } else { - secondPassLoops[loopNumber].Add(secondPassVertex); + var loopVertex = loopVertices[loopNumber]; + + for (var i = vertex.Outgoing.Count - 1; i >= 0; i--) { + if (vertex.Outgoing[i].LoopNumber == loopNumber) { + vertex.RemoveOutgoingAt(i); + } } - inLoopsCount++; + loopVertex.AddOutgoing(vertex); } if (version != _version) { - totalNodesCount = default; return false; } - - vertex.Index = -1; // Reset index, will use later } - // Break the loops so that its items can be iterated - foreach (var loop in secondPassLoops) { - // Sort loop items by amount of incoming connections - loop.Sort(WalkingVertex.FirstPassIncomingComparison); - - var counter = 0; - foreach (var secondPassVertex in loop) { - var vertex = secondPassVertex.FirstPass; - if (vertex.Index == -1) { - RemoveOutgoingLoopEdges(vertex, ref counter); - } - - if (version != _version) { - totalNodesCount = default; - return false; - } - } - } - - // Make first vertex from second pass loop (loop is sorted at this point) have incoming edges from vertices from first pass loop and set unique loop numbers - var outgoingVertices = new HashSet>(); - foreach (var loop in secondPassLoops) { - outgoingVertices.Clear(); - var startVertex = loop[0]; - - foreach (var secondPassVertex in loop) { - var firstPassVertex = secondPassVertex.FirstPass; - firstPassVertex.AddOutgoing(startVertex); - - foreach (var outgoingVertex in firstPassVertex.Outgoing) { - if (outgoingVertex.LoopNumber != firstPassVertex.LoopNumber) { - // Collect outgoing vertices to reference them from loop - outgoingVertices.Add(outgoingVertex); - } else if (outgoingVertex.SecondPass != null) { - // Copy outgoing edges to the second pass vertex - secondPassVertex.AddOutgoing(outgoingVertex.SecondPass); - } + // Connect dependencies to loop vertex + var outgoingLoopVertices = new HashSet>(); + foreach (var vertex in graph) { + outgoingLoopVertices.Clear(); + for (var i = vertex.Outgoing.Count - 1; i >= 0; i--) { + var outgoing = vertex.Outgoing[i]; + if (outgoing.IsInLoop && outgoing.LoopNumber != vertex.LoopNumber) { + var loopVertex = loopVertices[outgoing.LoopNumber]; + vertex.RemoveOutgoingAt(i); + outgoingLoopVertices.Add(loopVertex); } } - // Add outgoing edges to all second pass vertices to ensure that further analysis won't start until loop is fully analyzed - foreach (var secondPassVertex in loop) { - secondPassVertex.AddOutgoing(outgoingVertices); + if (outgoingLoopVertices.Count > 0) { + vertex.AddOutgoing(outgoingLoopVertices); } if (version != _version) { - totalNodesCount = default; return false; } - - loopsCount++; } - totalNodesCount = graph.Count + inLoopsCount; return true; } - private static void RemoveOutgoingLoopEdges(WalkingVertex vertex, ref int counter) { - vertex.Index = counter++; - for (var i = vertex.Outgoing.Count - 1; i >= 0; i--) { - var outgoing = vertex.Outgoing[i]; - if (outgoing.LoopNumber != vertex.LoopNumber) { - continue; - } - - if (outgoing.Index == -1) { - RemoveOutgoingLoopEdges(outgoing, ref counter); - } else if (outgoing.Index < vertex.Index) { - vertex.RemoveOutgoingAt(i); - } - } - } - private bool TryFindMissingDependencies(in ImmutableArray> vertices, in ImmutableArray> walkingGraph, int version, out ImmutableArray missingKeys) { var haveMissingDependencies = new bool[vertices.Count]; var queue = new Queue>(); @@ -581,9 +531,8 @@ private bool TryFindMissingDependencies(in ImmutableArray _dependencyResolver; private readonly ImmutableArray> _startingVertices; private readonly ImmutableArray _depths; - private readonly object _syncObj; + private readonly object _syncObj = new object(); private int _remaining; - private PriorityProducerConsumer> _ppc; + private PriorityProducerConsumer _ppc; public ImmutableArray MissingKeys { get; } public ImmutableArray AffectedValues { get; } @@ -623,10 +572,8 @@ public DependencyChainWalker(in DependencyResolver dependencyResol in ImmutableArray affectedValues, in ImmutableArray depths, in ImmutableArray missingKeys, - in int totalNodesCount, in int version) { - _syncObj = new object(); _dependencyResolver = dependencyResolver; _startingVertices = startingVertices; _depths = depths; @@ -634,17 +581,17 @@ public DependencyChainWalker(in DependencyResolver dependencyResol Version = version; MissingKeys = missingKeys; - _remaining = totalNodesCount; + _remaining = affectedValues.Count; } - public Task> GetNextAsync(CancellationToken cancellationToken) { - PriorityProducerConsumer> ppc; + public Task GetNextAsync(CancellationToken cancellationToken) { + PriorityProducerConsumer ppc; lock (_syncObj) { if (_ppc == null) { - _ppc = new PriorityProducerConsumer>(); + _ppc = new PriorityProducerConsumer(); foreach (var vertex in _startingVertices) { - _ppc.Produce(new DependencyChainNode(this, vertex, _depths[vertex.DependencyVertex.Index])); + _ppc.Produce(CreateNode(vertex)); } } @@ -654,7 +601,7 @@ public Task> GetNextAsync(CancellationToken cancell return ppc.ConsumeAsync(cancellationToken); } - public void MoveNext(WalkingVertex vertex) { + public void MoveNext(WalkingVertex vertex, bool loopAnalysis) { var verticesToProduce = new List>(); var isCompleted = false; lock (_syncObj) { @@ -664,7 +611,7 @@ public void MoveNext(WalkingVertex vertex) { continue; } - outgoing.DecrementIncoming(vertex.HasOnlyWalkedIncoming); + outgoing.DecrementIncoming(vertex.HasOnlyWalkedIncoming || loopAnalysis); if (outgoing.HasIncoming) { continue; } @@ -681,7 +628,7 @@ public void MoveNext(WalkingVertex vertex) { _ppc.Produce(null); } else { foreach (var toProduce in verticesToProduce) { - _ppc.Produce(new DependencyChainNode(this, toProduce, _depths[toProduce.DependencyVertex.Index])); + _ppc.Produce(CreateNode(toProduce)); } } } @@ -693,31 +640,68 @@ public bool IsValidVersion { } } } + + private IDependencyChainNode CreateNode(WalkingVertex vertex) { + if (vertex.DependencyVertex != null) { + return new SingleNode(this, vertex, _depths[vertex.DependencyVertex.Index]); + } + + var vertices = vertex.Outgoing; + var values = vertices.Select(v => v.DependencyVertex.Value).ToImmutableArray(); + var depth = vertices.Min(v => _depths[v.DependencyVertex.Index]); + var hasMissingDependencies = vertices.Any(v => v.HasMissingDependencies); + return new LoopNode(this, vertices, values, depth, hasMissingDependencies); + } } - private sealed class DependencyChainNode : IDependencyChainNode { + [DebuggerDisplay("{" + nameof(Value) + "}")] + private sealed class SingleNode : IDependencyChainSingleNode { private readonly WalkingVertex _vertex; private DependencyChainWalker _walker; public TValue Value => _vertex.DependencyVertex.Value; public int VertexDepth { get; } public bool HasMissingDependencies => _vertex.HasMissingDependencies; - public bool HasOnlyWalkedDependencies => _vertex.HasOnlyWalkedIncoming && _vertex.SecondPass == null; + public bool HasOnlyWalkedDependencies => _vertex.HasOnlyWalkedIncoming; public bool IsWalkedWithDependencies => _vertex.HasOnlyWalkedIncoming && _vertex.DependencyVertex.IsWalked; public bool IsValidVersion => _walker.IsValidVersion; - public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) { - _walker = walker; - _vertex = vertex; - VertexDepth = depth; - } + public SingleNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) => (_walker, _vertex, VertexDepth) = (walker, vertex, depth); + + public void MarkWalked() => _vertex.DependencyVertex.MarkWalked(); + + public void MoveNext() => Interlocked.Exchange(ref _walker, null)?.MoveNext(_vertex, loopAnalysis: false); + } + + [DebuggerDisplay("Loop: {_vertices.Count} nodes")] + private sealed class LoopNode : IDependencyChainLoopNode { + private readonly IReadOnlyList> _vertices; + private DependencyChainWalker _walker; + + public int VertexDepth { get; } + public bool HasMissingDependencies { get; } + public bool HasOnlyWalkedDependencies => _vertices.All(v => v.HasOnlyWalkedIncoming); + public bool IsWalkedWithDependencies => _vertices.All(v => v.HasOnlyWalkedIncoming && v.DependencyVertex.IsWalked); + public bool IsValidVersion => _walker.IsValidVersion; + + public ImmutableArray Values { get; } + + public LoopNode(DependencyChainWalker walker, IReadOnlyList> vertices, ImmutableArray values, int depth, bool hasMissingDependencies) + => (_walker, _vertices, Values, VertexDepth, HasMissingDependencies) = (walker, vertices, values, depth, hasMissingDependencies); public void MarkWalked() { - if (_vertex.SecondPass == null) { - _vertex.DependencyVertex.MarkWalked(); + foreach (var vertex in _vertices) { + vertex.DependencyVertex.MarkWalked(); } } - public void MoveNext() => Interlocked.Exchange(ref _walker, null)?.MoveNext(_vertex); + public void MoveNext() { + var walker = Interlocked.Exchange(ref _walker, null); + if (walker != null) { + foreach (var vertex in _vertices) { + walker.MoveNext(vertex, loopAnalysis: true); + } + } + } } } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs index 2f2270251..f0ef93f98 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyWalker.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Linq; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Collections; diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs index a7fca9a45..2532cb6e6 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs @@ -13,8 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Core.Collections; + namespace Microsoft.Python.Analysis.Dependencies { - internal interface IDependencyChainNode { + internal interface IDependencyChainNode { int VertexDepth { get; } /// @@ -33,8 +35,15 @@ internal interface IDependencyChainNode { /// Returns true if node version matches version of the walked graph /// bool IsValidVersion { get; } - TValue Value { get; } void MarkWalked(); void MoveNext(); } + + internal interface IDependencyChainSingleNode : IDependencyChainNode { + TValue Value { get; } + } + + internal interface IDependencyChainLoopNode : IDependencyChainNode { + ImmutableArray Values { get; } + } } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs index 87697bd27..98e68ef0c 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainWalker.cs @@ -23,6 +23,6 @@ internal interface IDependencyChainWalker { ImmutableArray AffectedValues { get; } int Version { get; } int Remaining { get; } - Task> GetNextAsync(CancellationToken cancellationToken); + Task GetNextAsync(CancellationToken cancellationToken); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/ImportExportWalker.cs b/src/Analysis/Ast/Impl/Dependencies/ImportExportWalker.cs new file mode 100644 index 000000000..4710d0ff8 --- /dev/null +++ b/src/Analysis/Ast/Impl/Dependencies/ImportExportWalker.cs @@ -0,0 +1,256 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Collections; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.OS; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Dependencies { + internal sealed class ImportExportWalker : PythonWalker { + private readonly Dictionary<(AnalysisModuleKey Module, string Name), IndexSpan> _imports; + private readonly Dictionary _exports; + private readonly PythonAst _ast; + private readonly IOSPlatform _platformService; + private readonly PathResolverSnapshot _pathResolver; + private readonly string _filePath; + private readonly bool _isTypeshed; + private int _depth; + + public IEnumerable<(AnalysisModuleKey Module, string Name, IndexSpan Location)> Imports + => _imports.Select(kvp => (kvp.Key.Module, kvp.Key.Name, kvp.Value)); + public IEnumerable<(string Name, IndexSpan Location)> Exports + => _exports.Select(kvp => (kvp.Key, kvp.Value)); + + public ImportExportWalker(PythonAst ast, IOSPlatform platformService, PathResolverSnapshot pathResolver, string filePath, bool isTypeshed) { + _imports = new Dictionary<(AnalysisModuleKey Module, string Name), IndexSpan>(); + _exports = new Dictionary(); + _ast = ast; + _platformService = platformService; + _pathResolver = pathResolver; + _isTypeshed = isTypeshed; + _filePath = filePath; + } + + public void Walk() => _ast.Walk(this); + + public override bool Walk(AssignmentStatement node) { + if (_depth == 0) { + HandleAssignment(node); + } + return base.Walk(node); + } + + private void HandleAssignment(AssignmentStatement node) { + foreach (var expr in node.Left.Select(s => s.RemoveParenthesis()).OfType()) { + AddExport(expr.Name, expr.IndexSpan); + } + + if (node.Right is MemberExpression me) { + AddImportIfModule(me); + } + } + + public override bool Walk(IfStatement node) => node.WalkIfWithSystemConditions(this, _ast.LanguageVersion, _platformService); + + public override bool Walk(ImportStatement expr) { + if (_depth > 0) { + return false; + } + + var len = Math.Min(expr.Names.Count, expr.AsNames.Count); + var forceAbsolute = expr.ForceAbsolute; + for (var i = 0; i < len; i++) { + var moduleImportExpression = expr.Names[i]; + var asNameExpression = expr.AsNames[i]; + + if (!string.IsNullOrEmpty(asNameExpression?.Name)) { + AddLastModuleImport(moduleImportExpression, asNameExpression, forceAbsolute); + } else { + AddAllImports(moduleImportExpression, forceAbsolute); + } + } + + return false; + } + + private void AddLastModuleImport(ModuleName importExpression, NameExpression importName, bool forceAbsolute) { + var result = _pathResolver.GetImportsFromAbsoluteName(_filePath, importExpression.Names.Select(n => n.Name), forceAbsolute); + if (result is ModuleImport mi) { + AddImport(mi, default, importName.IndexSpan); + } + } + + private void AddAllImports(ModuleName moduleImportExpression, bool forceAbsolute) { + var importNames = ImmutableArray.Empty; + + for (var i = 0; i < moduleImportExpression.Names.Count; i++) { + var nameExpression = moduleImportExpression.Names[i]; + importNames = importNames.Add(nameExpression.Name); + var result = _pathResolver.GetImportsFromAbsoluteName(_filePath, importNames, forceAbsolute); + if (result is ModuleImport mi && !mi.ModulePath.PathEquals(_filePath)) { + AddImport(mi, default, nameExpression.IndexSpan); + if (i == 0) { + AddExport(nameExpression.Name, nameExpression.IndexSpan); + } + } + } + } + + public override bool Walk(FromImportStatement expr) { + if (_depth > 0) { + return base.Walk(expr); + } + + var rootNames = expr.Root.Names; + if (rootNames.Count == 1 && rootNames[0].Name.EqualsOrdinal("__future__")) { + return base.Walk(expr); + } + + var imports = _pathResolver.FindImports(_filePath, expr); + if (!(imports is ModuleImport mi)) { + return base.Walk(expr); + } + + var names = expr.Names; + var asNames = expr.AsNames; + if (names.Count == 1 && names[0].Name == "*") { + AddImport(mi, default, names[0].IndexSpan); + return base.Walk(expr); + } + + for (var i = 0; i < names.Count; i++) { + var memberName = names[i].Name; + if (string.IsNullOrEmpty(memberName)) { + continue; + } + + var nameExpression = asNames[i] ?? names[i]; + if (mi.TryGetChildImport(nameExpression.Name, out var child) && child is ModuleImport childMi) { + AddImport(childMi, default, nameExpression.IndexSpan); + } else { + AddImport(mi, names[i].Name, nameExpression.IndexSpan); + } + + AddExport(nameExpression.Name, nameExpression.IndexSpan); + } + + return base.Walk(expr); + } + + public override bool Walk(MemberExpression expr) { + if (_depth == 0) { + AddImportIfModule(expr); + } + + return base.Walk(expr); + } + + public override bool Walk(ClassDefinition cd) { + if (_depth == 0 && !string.IsNullOrEmpty(cd.Name)) { + AddExport(cd.Name, cd.NameExpression.IndexSpan); + } + _depth++; + return base.Walk(cd); + } + + public override void PostWalk(ClassDefinition cd) { + _depth--; + base.PostWalk(cd); + } + + public override bool Walk(FunctionDefinition fd) { + if (_depth == 0 && !string.IsNullOrEmpty(fd.Name)) { + AddExport(fd.Name, fd.NameExpression.IndexSpan); + } + _depth++; + return base.Walk(fd); + } + + public override void PostWalk(FunctionDefinition fd) { + _depth--; + base.PostWalk(fd); + } + + private void AddExport(in string name, IndexSpan location) { + if (!_exports.TryGetValue(name, out var current) || current.Start > location.Start) { + _exports[name] = location; + } + } + + private void AddImportIfModule(in MemberExpression expr) { + var currentExpression = expr; + var memberExpressions = new Stack(); + memberExpressions.Push(currentExpression); + + while (currentExpression.Target is MemberExpression me) { + memberExpressions.Push(me); + currentExpression = me; + } + + if (!(currentExpression.Target is NameExpression ne)) { + return; + } + + var import = _pathResolver.GetModuleImportFromModuleName(ne.Name); + if (import == null) { + return; + } + + var moduleKey = new AnalysisModuleKey(import.Name, import.ModulePath, _isTypeshed); + IImportChildrenSource childrenSource = _pathResolver.GetModuleImportFromModuleName(moduleKey.Name); + if (childrenSource == null) { + return; + } + + while (memberExpressions.Count > 0) { + var expression = memberExpressions.Pop(); + + if (!childrenSource.TryGetChildImport(expression.Name, out var child)) { + AddImport(moduleKey, expression.Name, expression.IndexSpan); + return; + } + + if (child is IImportChildrenSource cs) { + childrenSource = cs; + } else { + return; + } + } + } + + private void AddImport(in ModuleImport moduleImport, in string name, in IndexSpan location) + => AddImport(new AnalysisModuleKey(moduleImport.FullName, moduleImport.ModulePath, _isTypeshed), name, location); + + private void AddImport(in AnalysisModuleKey key, in string name, in IndexSpan location) { + if (key.FilePath.PathEquals(_filePath)) { + return; + } + + if (_imports.TryGetValue((key, name), out var current) && current.Start <= location.Start) { + return; + } + + _imports[(key, name)] = location; + } + } +} diff --git a/src/Analysis/Ast/Impl/Dependencies/LocationLoopResolver.cs b/src/Analysis/Ast/Impl/Dependencies/LocationLoopResolver.cs new file mode 100644 index 000000000..8ce8bc6c2 --- /dev/null +++ b/src/Analysis/Ast/Impl/Dependencies/LocationLoopResolver.cs @@ -0,0 +1,134 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Collections; + +namespace Microsoft.Python.Analysis.Dependencies { + internal static class LocationLoopResolver { + public static ImmutableArray FindStartingItems(IEnumerable<(T From, int FromLocation, T To, int ToLocation)> edges) { + var itemToIndex = new Dictionary(); + var groupedEdges = new List>(); + var index = 0; + foreach (var (fromItem, fromLocation, toItem, toLocation) in edges) { + if (!itemToIndex.TryGetValue(fromItem, out var fromIndex)) { + fromIndex = index++; + groupedEdges.Add(new List<(int, int, int)>()); + itemToIndex[fromItem] = fromIndex; + } + + if (!itemToIndex.TryGetValue(toItem, out var toIndex)) { + toIndex = index++; + groupedEdges.Add(new List<(int, int, int)>()); + itemToIndex[toItem] = toIndex; + } + + groupedEdges[fromIndex].Add((fromLocation, toIndex, toLocation)); + } + + foreach (var group in groupedEdges) { + group.Sort(SortByFromLocation); + } + + var startingIndices = FindStartingIndices(groupedEdges); + return startingIndices.Select(i => itemToIndex.First(j => j.Value == i).Key).ToImmutableArray(); + + int SortByFromLocation((int FromLocation, int, int) x, (int FromLocation, int, int) y) => x.FromLocation.CompareTo(y.FromLocation); + } + + private static IEnumerable FindStartingIndices(List> groupedEdges) { + var walkedIndices = new int[groupedEdges.Count]; + var visited = new bool[groupedEdges.Count]; + var path = new Stack(); + var startingIndex = 0; + var allVisitedBeforeIndex = 0; + + while (startingIndex < groupedEdges.Count) { + if (visited[startingIndex]) { + if (startingIndex == allVisitedBeforeIndex) { + allVisitedBeforeIndex++; + } + + startingIndex++; + continue; + } + + for (var i = 0; i < walkedIndices.Length; i++) { + walkedIndices[i] = -1; + } + + path.Clear(); + + if (!IsWalkable(groupedEdges, startingIndex, walkedIndices, visited, path)) { + startingIndex++; + continue; + } + + for (var i = 0; i < walkedIndices.Length; i++) { + if (walkedIndices[i] != -1) { + visited[i] = true; + } + } + + yield return startingIndex; + startingIndex = allVisitedBeforeIndex; + } + } + + private static bool IsWalkable(in List> groupedEdges, in int startGroupIndex, in int[] walkedIndices, in bool[] visited, in Stack path) { + const int notVisited = -1; + var fromGroupIndex = startGroupIndex; + + while (true) { + var indexInFromGroup = ++walkedIndices[fromGroupIndex]; + var fromGroup = groupedEdges[fromGroupIndex]; + if (fromGroup.Count == indexInFromGroup) { + if (path.Count == 0) { + return true; + } + + fromGroupIndex = path.Pop(); + continue; + } + + var edge = fromGroup[indexInFromGroup]; + var toGroupIndex = edge.ToIndex; + if (visited[toGroupIndex]) { + continue; + } + + var indexInToGroup = walkedIndices[toGroupIndex]; + if (indexInToGroup == notVisited) { + path.Push(fromGroupIndex); + fromGroupIndex = toGroupIndex; + continue; + } + + var toGroup = groupedEdges[toGroupIndex]; + if (toGroup.Count == indexInToGroup) { + continue; + } + + var requiredPosition = edge.ToLocation; + var currentPosition = toGroup[indexInToGroup].FromLocation; + if (requiredPosition > currentPosition) { + return false; + } + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs index d3debee4f..041709891 100644 --- a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using Microsoft.Python.Core.Diagnostics; @@ -21,8 +20,6 @@ namespace Microsoft.Python.Analysis.Dependencies { [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] internal sealed class WalkingVertex { - public static Comparison> FirstPassIncomingComparison { get; } = (v1, v2) => v1.FirstPass._incomingCount.CompareTo(v2.FirstPass._incomingCount); - private readonly List> _outgoing; private bool _isSealed; private int _incomingCount; @@ -36,18 +33,21 @@ internal sealed class WalkingVertex { public bool HasOnlyWalkedIncoming => _walkedIncomingCount == 0; public bool HasMissingDependencies { get; private set; } - public WalkingVertex FirstPass { get; } - public WalkingVertex SecondPass { get; private set; } - public bool IsInLoop => LoopNumber >= 0; - public string DebuggerDisplay => DependencyVertex.DebuggerDisplay; + public string DebuggerDisplay => DependencyVertex?.DebuggerDisplay ?? "Loop node"; - public WalkingVertex(DependencyVertex vertex, WalkingVertex firstPass = null) { + public WalkingVertex(DependencyVertex vertex) { DependencyVertex = vertex; - FirstPass = firstPass; Index = -1; - LoopNumber = firstPass?.LoopNumber ?? -1; + LoopNumber = -1; + _outgoing = new List>(); + } + + public WalkingVertex(int loopNumber) { + DependencyVertex = default; + Index = -1; + LoopNumber = loopNumber; _outgoing = new List>(); } @@ -83,13 +83,6 @@ public void RemoveOutgoingAt(int index) { outgoingVertex._walkedIncomingCount--; } - public WalkingVertex CreateSecondPassVertex() { - CheckNotSealed(); - - SecondPass = new WalkingVertex(DependencyVertex, this); - return SecondPass; - } - public void Seal() => _isSealed = true; public void DecrementIncoming(bool isWalkedIncoming) { diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 9b2e259a0..35ff4e81f 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -85,6 +85,7 @@ public int DocumentCount { /// Optional file path, if different from the URI. public IDocument OpenDocument(Uri uri, string content, string filePath = null) { bool justOpened; + var created = false; IDocument document; lock (_lock) { var entry = FindDocument(uri); @@ -105,11 +106,16 @@ public IDocument OpenDocument(Uri uri, string content, string filePath = null) { ModuleType = moduleType }; entry = CreateDocument(mco); + created = true; } justOpened = TryOpenDocument(entry, content); document = entry.Document; } + if (created) { + _services.GetService().InvalidateAnalysis(document); + } + if (justOpened) { Opened?.Invoke(this, new DocumentEventArgs(document)); } diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs index f3ed567cb..e5ce15f5a 100644 --- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs @@ -13,8 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.Text; @@ -22,6 +25,35 @@ namespace Microsoft.Python.Analysis.Analyzer { public static class ScopeExtensions { + public static IEnumerable GetExportableVariableNames(this IGlobalScope scope) + // drop imported modules and typing + => scope.Variables + .Where(v => { + // Instances are always fine. + if (v.Value is IPythonInstance) { + return true; + } + + var valueType = v.Value?.GetPythonType(); + switch (valueType) { + case PythonModule _: + case IPythonFunctionType f when f.IsLambda(): + return false; // Do not re-export modules. + } + + if (scope.Module is TypingModule) { + return true; // Let typing module behave normally. + } + + // Do not re-export types from typing. However, do export variables + // assigned with types from typing. Example: + // from typing import Any # do NOT export Any + // x = Union[int, str] # DO export x + return !(valueType?.DeclaringModule is TypingModule) || v.Name != valueType.Name; + }) + .Select(v => v.Name) + .ToArray(); + public static IMember LookupNameInScopes(this IScope currentScope, string name, out IScope scope) { scope = null; foreach (var s in currentScope.EnumerateTowardsGlobal) { diff --git a/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs b/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs index 54ea7c16e..065a8c1c0 100644 --- a/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/VariableExtensions.cs @@ -13,17 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; namespace Microsoft.Python.Analysis { public static class VariableExtensions { public static T GetValue(this IVariable v) where T : class => v.Value as T; - - public static bool IsTypeInfo(this IVariable v) => v.Value is IPythonType; - public static bool IsTypeInfoOf(this IVariable v) where T : class, IPythonType => v.Value is T; - - public static bool IsInstance(this IVariable v) => v.Value is IPythonInstance; - public static bool IsInstanceOf(this IVariable v) where T: class, IPythonInstance => v.Value is T; } } diff --git a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj index 87fb189ef..be6afbfa1 100644 --- a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj +++ b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj @@ -1030,4 +1030,4 @@ - + \ No newline at end of file diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 1deda15bf..acb86ff0b 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -162,35 +162,8 @@ public virtual string Documentation { #region IMemberContainer public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; - public virtual IEnumerable GetMemberNames() { - // drop imported modules and typing. - return GlobalScope.Variables - .Where(v => { - // Instances are always fine. - if (v.Value is IPythonInstance) { - return true; - } - - var valueType = v.Value?.GetPythonType(); - switch (valueType) { - case PythonModule _: - case IPythonFunctionType f when f.IsLambda(): - return false; // Do not re-export modules. - } + public virtual IEnumerable GetMemberNames() => GlobalScope.GetExportableVariableNames(); - if (this is TypingModule) { - return true; // Let typing module behave normally. - } - - // Do not re-export types from typing. However, do export variables - // assigned with types from typing. Example: - // from typing import Any # do NOT export Any - // x = Union[int, str] # DO export x - return !(valueType?.DeclaringModule is TypingModule) || v.Name != valueType.Name; - }) - .Select(v => v.Name) - .ToArray(); - } #endregion #region ILocatedMember @@ -348,7 +321,7 @@ private void Parse(CancellationToken cancellationToken) { int version; Parser parser; - //Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); + // Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name} ({ModuleType})"); lock (_syncObj) { version = _buffer.Version; @@ -364,7 +337,7 @@ private void Parse(CancellationToken cancellationToken) { var ast = parser.ParseFile(Uri); - //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); + // Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name} ({ModuleType})"); lock (_syncObj) { cancellationToken.ThrowIfCancellationRequested(); @@ -528,7 +501,6 @@ private void InitializeContent(string content, int version) { Parse(); } } - Services.GetService().InvalidateAnalysis(this); } private void SetOrLoadContent(string content) { diff --git a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs index d674c2b8c..323021deb 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonVariableModule.cs @@ -53,6 +53,7 @@ internal sealed class PythonVariableModule : LocatedMember, IPythonModule, IEqua public override PythonMemberType MemberType => PythonMemberType.Module; public bool IsPersistent => Module?.IsPersistent == true; public bool IsTypeshed => Module?.IsTypeshed == true; + public IEnumerable ChildrenNames => _children.Keys; public PythonVariableModule(string name, IPythonInterpreter interpreter) : base(null) { Name = name; @@ -69,7 +70,7 @@ public PythonVariableModule(IPythonModule module): base(module) { public void AddChildModule(string memberName, PythonVariableModule module) => _children[memberName] = module; public IMember GetMember(string name) => _children.TryGetValue(name, out var module) ? module : Module?.GetMember(name); - public IEnumerable GetMemberNames() => Module != null ? Module.GetMemberNames().Concat(_children.Keys).Distinct() : _children.Keys; + public IEnumerable GetMemberNames() => Module != null ? Module.GetMemberNames().Concat(ChildrenNames).Distinct() : ChildrenNames; public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => GetMember(memberName); public IMember Index(IPythonInstance instance, IArgumentSet args) => Interpreter.UnknownType; @@ -78,7 +79,7 @@ public PythonVariableModule(IPythonModule module): base(module) { public bool Equals(IPythonModule other) => other is PythonVariableModule module && Name.EqualsOrdinal(module.Name); public override bool Equals(object obj) => Equals(obj as IPythonModule); - public override int GetHashCode() => 0; + public override int GetHashCode() => Name.GetHashCode(); #region ILocationConverter public SourceLocation IndexToLocation(int index) => (Module as ILocationConverter)?.IndexToLocation(index) ?? default; diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 4329daec3..dcceda246 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -91,7 +91,9 @@ protected override IPythonModule CreateModule(string name) { if (!moduleImport.IsPersistent) { // If there is a stub, make sure it is loaded and attached // First check stub next to the module. - if (!TryCreateModuleStub(name, moduleImport.ModulePath, out stub)) { + if (TryCreateModuleStub(name, moduleImport.ModulePath, out stub)) { + Analyzer.InvalidateAnalysis(stub); + } else { // If nothing found, try Typeshed. stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); } @@ -186,7 +188,7 @@ private async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancella // Add built-in module names var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); - var value = (builtinModuleNamesMember as IVariable)?.Value ?? builtinModuleNamesMember; + var value = builtinModuleNamesMember is IVariable variable ? variable.Value : builtinModuleNamesMember; if (value.TryGetConstant(out var s)) { var builtinModuleNames = s.Split(',').Select(n => n.Trim()); PathResolver.SetBuiltins(builtinModuleNames); diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 0a3782b24..52112e7bf 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -18,6 +18,7 @@ using System.IO; using System.Linq; using System.Threading; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Core.Interpreter; @@ -26,13 +27,13 @@ using Microsoft.Python.Core.Collections; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; -using Microsoft.Python.Core.Services; namespace Microsoft.Python.Analysis.Modules.Resolution { internal abstract class ModuleResolutionBase { protected IServiceContainer Services { get; } protected IFileSystem FileSystem { get; } protected IPythonInterpreter Interpreter { get; } + protected IPythonAnalyzer Analyzer { get; } protected ILogger Log { get; } protected ConcurrentDictionary Modules { get; } = new ConcurrentDictionary(); @@ -49,6 +50,7 @@ protected ModuleResolutionBase(string root, IServiceContainer services) { FileSystem = services.GetService(); Interpreter = services.GetService(); + Analyzer = services.GetService(); StubCache = services.GetService(); Log = services.GetService(); } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index 43f45cd2e..0c42ccbcd 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -45,23 +45,40 @@ public TypeshedResolution(string root, IServiceContainer services) : base(root, } protected override IPythonModule CreateModule(string name) { + if (!TryCreateStubModule(name, out var module)) { + return null; + } + + Analyzer.InvalidateAnalysis(module); + return module; + + } + + private bool TryCreateStubModule(string name, out IPythonModule module) { + module = null; var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name); if (moduleImport != null) { if (moduleImport.IsCompiled) { Log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath); - return null; + return false; } - return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, Services); + + module = new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, Services); + return true; } var i = name.IndexOf('.'); if (i == 0) { Debug.Fail("Invalid module name"); - return null; + return false; } var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => FileSystem.FileExists(p)); - return stubPath != null ? new StubPythonModule(name, stubPath, true, Services) : null; + if (stubPath != null) { + module = new StubPythonModule(name, stubPath, true, Services); + return true; + } + return false; } public Task ReloadAsync(CancellationToken cancellationToken = default) { diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index 53060d859..88a867020 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -23,6 +23,8 @@ public Location(IPythonModule module, IndexSpan indexSpan = default) { IndexSpan = indexSpan; } + public void Deconstruct(out IPythonModule module, out IndexSpan indexSpan) => (module, indexSpan) = (Module, IndexSpan); + public IPythonModule Module { get; } public IndexSpan IndexSpan { get; } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 708f254d4..9aa271d52 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -348,7 +348,7 @@ private IEnumerable DisambiguateBases(IEnumerable base // Get locally declared variable, make sure it is a declaration // and that it declared a class. var lv = scope.Variables[b.Name]; - if (lv.Source != VariableSource.Import && lv.Value is IPythonClassType cls && cls.IsDeclaredAfterOrAt(this.Location)) { + if (lv.Source != VariableSource.Import && lv.Value is IPythonClassType cls && cls.IsDeclaredAfterOrAt(Location)) { // There is a declaration with the same name, but it appears later in the module. Use the import. if (!importedType.IsUnknown()) { newBases.Add(importedType); diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index e394de6a2..831a7273d 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Symbols; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -128,6 +129,16 @@ public IMember Call(IArgumentSet args, IPythonType self) { if (!rt.IsUnknown()) { return rt; } + if (StaticReturnValue == null && !string.IsNullOrEmpty(_returnDocumentation) && FunctionDefinition?.ReturnAnnotation != null) { + // There is return documentation but no static return value. + // This may happen if function is inside module circular + // dependency loop. Try and re-evaluate now. + var returnValue = FunctionEvaluator.GetReturnValueFromAnnotation(args.Eval as ExpressionEval, FunctionDefinition.ReturnAnnotation); + if (returnValue != null) { + SetReturnValue(returnValue, true); + return returnValue; + } + } } return GetSpecificReturnType(self as IPythonClassType, args); diff --git a/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs b/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs index 60aa3e46c..12b50a2f2 100644 --- a/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/EmptyGlobalScope.cs @@ -28,6 +28,7 @@ public EmptyGlobalScope(IPythonModule module) { public IPythonModule Module { get; } public string Name => string.Empty; + public PythonAst Ast => Module.Analysis.Ast; public ScopeStatement Node => Module.Analysis.Ast; public IScope OuterScope => null; public IGlobalScope GlobalScope { get; } diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index 0316402cf..ed323da8d 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -15,6 +15,9 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core.Collections; +using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Values { @@ -41,14 +44,14 @@ private void DeclareBuiltinVariables() { var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); var objectType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - DeclareVariable("__debug__", boolType, VariableSource.Builtin, location); - DeclareVariable("__doc__", strType, VariableSource.Builtin, location); - DeclareVariable("__file__", strType, VariableSource.Builtin, location); - DeclareVariable("__name__", strType, VariableSource.Builtin, location); - DeclareVariable("__package__", strType, VariableSource.Builtin, location); - DeclareVariable("__path__", listType, VariableSource.Builtin, location); - DeclareVariable("__dict__", dictType, VariableSource.Builtin, location); - DeclareVariable("__spec__", objectType, VariableSource.Builtin, location); + DeclareBuiltinVariable("__debug__", boolType, location); + DeclareBuiltinVariable("__doc__", strType, location); + DeclareBuiltinVariable("__file__", strType, location); + DeclareBuiltinVariable("__name__", strType, location); + DeclareBuiltinVariable("__package__", strType, location); + DeclareBuiltinVariable("__path__", listType, location); + DeclareBuiltinVariable("__dict__", dictType, location); + DeclareBuiltinVariable("__spec__", objectType, location); } } } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index 6150e812b..2a7d967e7 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -19,6 +19,7 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Values { @@ -33,6 +34,8 @@ internal class Scope : IScope { private List _childScopes; public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { + Check.ArgumentNotNull(nameof(module), module); + OuterScope = outerScope; Module = module; if (node != null) { @@ -104,22 +107,25 @@ private void DeclareBuiltinVariables() { var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin, location); + DeclareBuiltinVariable("__name__", strType, location); if (Node is FunctionDefinition) { var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); var tupleType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Tuple); - VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin, location); - VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin, location); - VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin, location); - VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin, location); - VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin, location); - VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin, location); - VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin, location); + DeclareBuiltinVariable("__closure__", tupleType, location); + DeclareBuiltinVariable("__code__", objType, location); + DeclareBuiltinVariable("__defaults__", tupleType, location); + DeclareBuiltinVariable("__dict__", dictType, location); + DeclareBuiltinVariable("__doc__", strType, location); + DeclareBuiltinVariable("__func__", objType, location); + DeclareBuiltinVariable("__globals__", dictType, location); } else if (Node is ClassDefinition) { - VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin, location); + DeclareBuiltinVariable("__self__", objType, location); } } + + protected void DeclareBuiltinVariable(string name, IPythonType type, Location location) + => VariableCollection.DeclareVariable(name, type, VariableSource.Builtin, location); } } diff --git a/src/Analysis/Ast/Test/DependencyResolverTests.cs b/src/Analysis/Ast/Test/DependencyResolverTests.cs index 92effbbca..a78cfdde3 100644 --- a/src/Analysis/Ast/Test/DependencyResolverTests.cs +++ b/src/Analysis/Ast/Test/DependencyResolverTests.cs @@ -13,12 +13,14 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Core.Collections; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -36,16 +38,17 @@ public void TestInitialize() public void Cleanup() => TestEnvironmentImpl.TestCleanup(); // ReSharper disable StringLiteralTypo + // Square brackets mean that nodes can be walked in parallel. Parentheses mean that nodes are in loop. [DataRow("A:BC|B:C|C", "CBA", "A")] [DataRow("C|A:BC|B:C", "CBA", "A")] - [DataRow("C|B:AC|A:BC", "CBABA", "A")] - [DataRow("A:CE|B:A|C:B|D:B|E", "[CE]ABCABD", "D")] - [DataRow("A:D|B:DA|C:BA|D:AE|E", "[AE]DADBC", "C")] - [DataRow("A:C|C:B|B:A|D:AF|F:CE|E:BD", "ABCABCDEFDEF", "F")] - [DataRow("A:BC|B:AC|C:BA|D:BC", "ACBACBD", "D")] + [DataRow("C|B:AC|A:BC", "C(BA)", "A")] + [DataRow("A:CE|B:A|C:B|D:B|E", "E(ACB)D", "D")] + [DataRow("A:D|B:DA|C:BA|D:AE|E", "E(AD)BC", "C")] + [DataRow("A:C|C:B|B:A|D:AF|F:CE|E:BD", "(ACB)(DFE)", "F")] + [DataRow("A:BC|B:AC|C:BA|D:BC", "(ABC)D", "D")] [DataRow("A|B|C|D:AB|E:BC", "[ABC][DE]", "D|E")] - [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "[CE]ABCAB[DF]", "D|F")] - [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "DFEDFE[ABC]", "A|B|C")] + [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "E(ACB)[FD]", "D|F")] + [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "(DEF)[ABC]", "A|B|C")] // ReSharper restore StringLiteralTypo [DataTestMethod] public void ChangeValue(string input, string output, string root) { @@ -61,7 +64,7 @@ public void ChangeValue(string input, string output, string root) { var walker = resolver.CreateWalker(); var result = new StringBuilder(); - var tasks = new List>>(); + var tasks = new List>(); while (walker.Remaining > 0) { var nodeTask = walker.GetNextAsync(default); if (!nodeTask.IsCompleted) { @@ -70,7 +73,7 @@ public void ChangeValue(string input, string output, string root) { } foreach (var task in tasks) { - result.Append(task.Result.Value[0]); + AppendFirstChar(result, task.Result); task.Result.MarkWalked(); task.Result.MoveNext(); } @@ -86,7 +89,7 @@ public void ChangeValue(string input, string output, string root) { result.ToString().Should().Be(output); } - + [TestMethod] public async Task ChangeValue_ChangeToIdentical() { var resolver = new DependencyResolver(); @@ -98,8 +101,8 @@ public async Task ChangeValue_ChangeToIdentical() { var result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + AppendFirstChar(result, node); + node.Should().HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); } @@ -112,8 +115,8 @@ public async Task ChangeValue_ChangeToIdentical() { result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + AppendFirstChar(result, node); + node.Should().HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); } @@ -133,8 +136,8 @@ public async Task ChangeValue_TwoChanges() { var result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + AppendFirstChar(result, node); + node.Should().HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); } @@ -148,8 +151,8 @@ public async Task ChangeValue_TwoChanges() { result = new StringBuilder(); while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + AppendFirstChar(result, node); + node.Should().HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); } @@ -164,31 +167,30 @@ public async Task ChangeValue_MissingKeys() { resolver.ChangeValue("B", "B", false); resolver.ChangeValue("C", "C:D", true, "D"); var walker = resolver.CreateWalker(); - - var result = new StringBuilder(); + var node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("B") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("C:D") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); walker.MissingKeys.Should().Equal("D"); - result.ToString().Should().Be("BC"); resolver.ChangeValue("D", "D", false); walker = resolver.CreateWalker(); - result = new StringBuilder(); - result.Append((await walker.GetNextAsync(default)).Value[0]); - result.Append((await walker.GetNextAsync(default)).Value[0]); - walker.MissingKeys.Should().BeEmpty(); - result.ToString().Should().Be("AD"); + + node = await walker.GetNextAsync(default); + node.Should().HaveSingleValue("A:B"); + + node = await walker.GetNextAsync(default); + node.Should().HaveSingleValue("D"); } [TestMethod] @@ -200,9 +202,9 @@ public async Task ChangeValue_Add() { var walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("B", "D"); var node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BD"); - node.HasMissingDependencies.Should().BeTrue(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A:BD") + .And.HaveMissingDependencies() + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -212,46 +214,46 @@ public async Task ChangeValue_Add() { resolver.ChangeValue("B", "B", false); walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("D"); - + node = await walker.GetNextAsync(default); - node.Value.Should().Be("B"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("B") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); - node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BD"); - node.HasMissingDependencies.Should().BeTrue(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node = await walker.GetNextAsync(default); + node.Should().HaveSingleValue("A:BD") + .And.HaveMissingDependencies() + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); walker.Remaining.Should().Be(0); - + // Add D resolver.ChangeValue("D", "D:C", false, "C"); walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("C"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("C") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); - node = await walker.GetNextAsync(default); - node.Value.Should().Be("D:C"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node = await walker.GetNextAsync(default); + node.Should().HaveSingleValue("D:C") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BD"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A:BD") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -259,7 +261,7 @@ public async Task ChangeValue_Add() { } [TestMethod] - public async Task ChangeValue_Add_ParallelWalkers() { + public async Task ChangeValue_Add_ParallelWalkers() { var resolver = new DependencyResolver(); resolver.ChangeValue("A", "A:BD", true, "B", "D"); resolver.ChangeValue("B", "B:C", false, "C"); @@ -269,9 +271,9 @@ public async Task ChangeValue_Add_ParallelWalkers() { walker.MissingKeys.Should().Equal("D"); var node = await walker.GetNextAsync(default); - node.Value.Should().Be("C"); - node.HasMissingDependencies.Should().BeFalse(); - node.IsValidVersion.Should().BeTrue(); + node.Should().HaveSingleValue("C") + .And.HaveNoMissingDependencies() + .And.HaveValidVersion(); // Add D resolver.ChangeValue("D", "D:C", false, "C"); @@ -279,24 +281,24 @@ public async Task ChangeValue_Add_ParallelWalkers() { newWalker.MissingKeys.Should().BeEmpty(); // MarkWalked node from old walker - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsValidVersion.Should().BeFalse(); + node.Should().HaveOnlyWalkedDependencies() + .And.HaveInvalidVersion(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("B:C"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsValidVersion.Should().BeFalse(); + node.Should().HaveSingleValue("B:C") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies() + .And.HaveInvalidVersion(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BD"); - node.HasMissingDependencies.Should().BeTrue(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsValidVersion.Should().BeFalse(); + node.Should().HaveSingleValue("A:BD") + .And.HaveMissingDependencies() + .And.HaveOnlyWalkedDependencies() + .And.HaveInvalidVersion(); node.MarkWalked(); node.MoveNext(); @@ -304,34 +306,34 @@ public async Task ChangeValue_Add_ParallelWalkers() { // Walk new walker node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("C"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsValidVersion.Should().BeTrue(); + node.Should().HaveSingleValue("C") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies() + .And.HaveValidVersion(); node.MarkWalked(); node.MoveNext(); node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("B:C"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsValidVersion.Should().BeTrue(); + node.Should().HaveSingleValue("B:C") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies() + .And.HaveValidVersion(); node.MarkWalked(); node.MoveNext(); node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("D:C"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsValidVersion.Should().BeTrue(); + node.Should().HaveSingleValue("D:C") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies() + .And.HaveValidVersion(); node.MarkWalked(); node.MoveNext(); node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("A:BD"); - node.HasMissingDependencies.Should().BeFalse(); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsValidVersion.Should().BeTrue(); + node.Should().HaveSingleValue("A:BD") + .And.HaveNoMissingDependencies() + .And.HaveOnlyWalkedDependencies() + .And.HaveValidVersion(); node.MarkWalked(); node.MoveNext(); @@ -351,87 +353,34 @@ public async Task ChangeValue_PartiallyWalkLoop() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); - node.Value.Should().Be("E"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Value.Should().Be("B:CE"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.Should().HaveSingleValue("E") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("D:BE"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Value.Should().Be("C:DE"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.Should().HaveLoopValues("B:CE", "C:DE", "D:BE") + .And.HaveNonWalkedDependencies(); node.MarkWalked(); node.MoveNext(); // Create new walker var newWalker = resolver.CreateWalker(); - // Mark vertex walked as it would've in parallel + // Mark vertex walked as it would've been in parallel + // Loops are always walked fully. node = await walker.GetNextAsync(default); - node.Value.Should().Be("B:CE"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsWalkedWithDependencies.Should().BeFalse(); + node.Should().HaveSingleValue("A:B") + .And.HaveOnlyWalkedDependencies() + .And.NotBeWalkedWithDependencies(); node.MarkWalked(); node.MoveNext(); // Now iterate with new walker node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("B:CE"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); - node.IsWalkedWithDependencies.Should().BeTrue(); - node.MarkWalked(); - node.MoveNext(); - - node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("D:BE"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); - node.IsWalkedWithDependencies.Should().BeFalse(); - node.MarkWalked(); - node.MoveNext(); - - node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("C:DE"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); - node.IsWalkedWithDependencies.Should().BeFalse(); - node.MarkWalked(); - node.MoveNext(); - - node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("B:CE"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsWalkedWithDependencies.Should().BeTrue(); - node.MarkWalked(); - node.MoveNext(); - - node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("D:BE"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsWalkedWithDependencies.Should().BeFalse(); - node.MarkWalked(); - node.MoveNext(); - - node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("C:DE"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsWalkedWithDependencies.Should().BeFalse(); - node.MarkWalked(); - node.MoveNext(); - - node = await newWalker.GetNextAsync(default); - node.Value.Should().Be("A:B"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.IsWalkedWithDependencies.Should().BeFalse(); + node.Should().HaveSingleValue("A:B") + .And.HaveOnlyWalkedDependencies() + .And.BeWalkedWithDependencies(); node.MarkWalked(); node.MoveNext(); @@ -448,20 +397,20 @@ public async Task ChangeValue_Remove() { var walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); - node.Value.Should().Be("C"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("C") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("B:C"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("B:C") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BC"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A:BC") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -470,8 +419,8 @@ public async Task ChangeValue_Remove() { walker.MissingKeys.Should().Equal("B"); node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BC"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A:BC") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -488,7 +437,7 @@ public async Task ChangeValue_ChangeChangeRemove() { var walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("D"); walker.AffectedValues.Should().Equal("A:B", "B:C", "C:AD"); - walker.Remaining.Should().Be(6); + walker.Remaining.Should().Be(3); //resolver.ChangeValue("D", "D:B", true, "B"); resolver.ChangeValue("A", "A", true); @@ -500,14 +449,14 @@ public async Task ChangeValue_ChangeChangeRemove() { walker.Remaining.Should().Be(2); var node = await walker.GetNextAsync(default); - node.Value.Should().Be("A"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("C:AD"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("C:AD") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -525,41 +474,11 @@ public async Task ChangeValue_RemoveFromLoop() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:B"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.Should().HaveLoopValues("A:B", "B:C", "C:A") + .And.HaveNonWalkedDependencies(); node.MarkWalked(); node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Value.Should().Be("C:A"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Value.Should().Be("B:C"); - node.HasOnlyWalkedDependencies.Should().BeFalse(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:B"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Value.Should().Be("C:A"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.MarkWalked(); - node.MoveNext(); - - node = await walker.GetNextAsync(default); - node.Value.Should().Be("B:C"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); - node.MarkWalked(); - node.MoveNext(); - + walker.Remaining.Should().Be(0); resolver.Remove("B"); @@ -567,8 +486,8 @@ public async Task ChangeValue_RemoveFromLoop() { walker.MissingKeys.Should().Equal("B"); node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:B"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A:B") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -586,26 +505,26 @@ public async Task ChangeValue_RemoveKeys() { var walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); - node.Value.Should().Be("D"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("D") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("C:D"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("C:D") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("B:C"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("B:C") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BC"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A:BC") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -614,14 +533,14 @@ public async Task ChangeValue_RemoveKeys() { walker.MissingKeys.Should().Equal("B", "D"); node = await walker.GetNextAsync(default); - node.Value.Should().Be("C:D"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("C:D") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); node = await walker.GetNextAsync(default); - node.Value.Should().Be("A:BC"); - node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.Should().HaveSingleValue("A:BC") + .And.HaveOnlyWalkedDependencies(); node.MarkWalked(); node.MoveNext(); @@ -639,22 +558,33 @@ public async Task ChangeValue_Skip() { var walker = resolver.CreateWalker(); var result = new StringBuilder(); var node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); + AppendFirstChar(result, node); node.MoveNext(); - + node = await walker.GetNextAsync(default); - result.Append(node.Value[0]); + AppendFirstChar(result, node); node.MoveNext(); - + result.ToString().Should().Be("BD"); resolver.ChangeValue("D", "D", false); walker = resolver.CreateWalker(); result = new StringBuilder(); - result.Append((await walker.GetNextAsync(default)).Value[0]); - result.Append((await walker.GetNextAsync(default)).Value[0]); + AppendFirstChar(result, await walker.GetNextAsync(default)); + AppendFirstChar(result, await walker.GetNextAsync(default)); result.ToString().Should().Be("BD"); } + + private static StringBuilder AppendFirstChar(StringBuilder sb, IDependencyChainNode node) { + switch (node) { + case IDependencyChainSingleNode single: + return sb.Append(single.Value[0]); + case IDependencyChainLoopNode loop: + return sb.Append($"({new string(loop.Values.Select(v => v[0]).ToArray())})"); + default: + throw new InvalidOperationException(); + } + } } } diff --git a/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs b/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs index 9e4d86965..70de139c9 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/AssertionsFactory.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System.Diagnostics.CodeAnalysis; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.Text; @@ -21,7 +22,7 @@ namespace Microsoft.Python.Analysis.Tests.FluentAssertions { [ExcludeFromCodeCoverage] internal static class AssertionsFactory { - public static ScopeAssertions Should(this IScope scope) => new ScopeAssertions(scope); + public static DependencyChainNodeAssertions Should(this IDependencyChainNode node) => new DependencyChainNodeAssertions(node); public static MemberAssertions Should(this IMember member) => new MemberAssertions(member); public static PythonFunctionAssertions Should(this IPythonFunctionType f) => new PythonFunctionAssertions(f); @@ -33,6 +34,7 @@ internal static class AssertionsFactory { public static RangeAssertions Should(this Range? range) => new RangeAssertions(range); + public static ScopeAssertions Should(this IScope scope) => new ScopeAssertions(scope); public static SourceSpanAssertions Should(this SourceSpan span) => new SourceSpanAssertions(span); public static SourceSpanAssertions Should(this SourceSpan? span) => new SourceSpanAssertions(span.Value); } diff --git a/src/Analysis/Ast/Test/FluentAssertions/DependencyChainNodeAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/DependencyChainNodeAssertions.cs new file mode 100644 index 000000000..d628ac178 --- /dev/null +++ b/src/Analysis/Ast/Test/FluentAssertions/DependencyChainNodeAssertions.cs @@ -0,0 +1,136 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Primitives; +using Microsoft.Python.Analysis.Dependencies; +using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; + +namespace Microsoft.Python.Analysis.Tests.FluentAssertions { + internal sealed class DependencyChainNodeAssertions : ReferenceTypeAssertions { + public DependencyChainNodeAssertions(IDependencyChainNode node) { + Subject = node; + } + + protected override string Identifier => nameof(IDependencyChainNode); + + [CustomAssertion] + public AndConstraint HaveSingleValue(T value, string because = "", params object[] reasonArgs) { + var currentStateMessage = Subject == null ? "null" : "loop node"; + + Execute.Assertion.ForCondition(Subject is IDependencyChainSingleNode) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected [{typeof(T)}] node to be single node{{reason}}, but it is {currentStateMessage}"); + + var actual = ((IDependencyChainSingleNode)Subject).Value; + Execute.Assertion.ForCondition(Equals(actual, value)) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected [{typeof(T)}] node to have value {value}{{reason}}, but it has {actual}"); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint HaveLoopValues(params T[] values) => HaveLoopValues(values, string.Empty); + + [CustomAssertion] + public AndConstraint HaveLoopValues(T[] values, string because = "", params object[] reasonArgs) { + var currentStateMessage = Subject == null ? "null" : "loop node"; + + Execute.Assertion.ForCondition(Subject is IDependencyChainLoopNode) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected [{typeof(T)}] node to be loop node{{reason}}, but it is {currentStateMessage}"); + + var actual = ((IDependencyChainLoopNode)Subject).Values.ToArray(); + var errorMessage = GetAssertCollectionOnlyContainsMessage(actual, values, "loop node", "value", "values"); + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(string.Empty, string.Empty) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint HaveOnlyWalkedDependencies(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(Subject.HasOnlyWalkedDependencies) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to have only walked dependencies{{reason}}"); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint HaveNonWalkedDependencies(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(!Subject.HasOnlyWalkedDependencies) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to have non-walked dependencies{{reason}}"); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint BeWalkedWithDependencies(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(Subject.IsWalkedWithDependencies) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to be walked with dependencies{{reason}}"); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint NotBeWalkedWithDependencies(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(!Subject.IsWalkedWithDependencies) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to not be walked with dependencies{{reason}}"); + + return new AndConstraint(this); + } + + public AndConstraint HaveMissingDependencies(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(Subject.HasMissingDependencies) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to have missing dependencies{{reason}}"); + + return new AndConstraint(this); + } + + public AndConstraint HaveNoMissingDependencies(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(!Subject.HasMissingDependencies) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to have no missing dependencies{{reason}}"); + + return new AndConstraint(this); + } + + public AndConstraint HaveValidVersion(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(Subject.IsValidVersion) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to have valid version{{reason}}"); + + return new AndConstraint(this); + } + + public AndConstraint HaveInvalidVersion(string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(!Subject.IsValidVersion) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {Subject} to have invalid version{{reason}}"); + + return new AndConstraint(this); + } + } +} diff --git a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs index 1f6b10c08..57770ed7b 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/MemberAssertions.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; namespace Microsoft.Python.Analysis.Tests.FluentAssertions { @@ -126,59 +127,62 @@ public AndConstraint HaveSameMemberNamesAs(IMember member) { return HaveMembers(((IMemberContainer)member).GetMemberNames(), string.Empty); } - public void HaveSameMembersAs(IMember other) { - other.Should().BeAssignableTo(); - var otherContainer = (IMemberContainer)other; + public void HaveSameMembersAs(IMember expected, string because = "", params object[] becauseArgs) { + var expectedContainer = expected.Should().BeAssignableTo().Which; var subjectType = Subject.GetPythonType(); - var subjectMemberNames = subjectType.GetMemberNames().ToArray(); - var otherMemberNames = otherContainer.GetMemberNames().ToArray(); + var actualNames = subjectType.GetMemberNames().ToArray(); + var expectedNames = expectedContainer.GetMemberNames().ToArray(); - var missingNames = otherMemberNames.Except(subjectMemberNames).ToArray(); - var extraNames = subjectMemberNames.Except(otherMemberNames).ToArray(); + var errorMessage = GetAssertCollectionOnlyContainsMessage(actualNames, expectedNames, GetQuotedName(Subject), "member", "members"); - Debug.Assert(missingNames.Length == 0); - missingNames.Should().BeEmpty("Subject has missing names: ", missingNames); + var assertion = Execute.Assertion.BecauseOf(because, becauseArgs); - Debug.Assert(extraNames.Length == 0); - extraNames.Should().BeEmpty("Subject has extra names: ", extraNames); + assertion.ForCondition(errorMessage == null).FailWith(errorMessage); - foreach (var n in subjectMemberNames.Except(Enumerable.Repeat("__base__", 1))) { - var subjectMember = subjectType.GetMember(n); - var otherMember = otherContainer.GetMember(n); - var subjectMemberType = subjectMember.GetPythonType(); - var otherMemberType = otherMember.GetPythonType(); + foreach (var n in actualNames.Except(Enumerable.Repeat("__base__", 1))) { + var actualMember = subjectType.GetMember(n); + var expectedMember = expectedContainer.GetMember(n); + var actualMemberType = actualMember.GetPythonType(); + var expectedMemberType = expectedMember.GetPythonType(); // PythonConstant, PythonUnicodeStrings... etc are mapped to instances. - if (subjectMember is IPythonInstance) { - otherMember.Should().BeAssignableTo(); + if (expectedMember is IPythonInstance && !expectedMember.IsUnknown()) { + assertion.ForCondition(actualMember is IPythonInstance) + .FailWith($"Expected '{GetName(subjectType)}.{n}' to implement IPythonInstance{{reason}}, but its type is {actualMember.GetType().FullName}"); } - subjectMemberType.MemberType.Should().Be(otherMemberType.MemberType, $"Type name: {subjectMemberType.Name}"); - //Debug.Assert(subjectMemberType.MemberType == otherMemberType.MemberType); + assertion.ForCondition(actualMemberType.MemberType == expectedMemberType.MemberType) + .FailWith($"Expected '{GetName(subjectType)}.{n}' to have MemberType {expectedMemberType.MemberType}{{reason}}, but it has MemberType {actualMemberType.MemberType}"); - if (subjectMemberType is IPythonClassType subjectClass) { - var otherClass = otherMemberType as IPythonClassType; - otherClass.Should().NotBeNull(); + if (expectedMemberType is IPythonClassType) { + assertion.ForCondition(actualMemberType is IPythonClassType) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to implement IPythonClassType{{reason}}, but python type is {actualMemberType.GetType().FullName}"); + } + + if (expectedMemberType is IGenericType expectedGenericType) { + assertion.ForCondition(actualMemberType is IGenericType) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to implement IGenericType{{reason}}, but python type is {actualMemberType.GetType().FullName}"); - if (subjectClass is IGenericType gt) { - otherClass.Should().BeAssignableTo(); - otherClass.IsGeneric.Should().Be(gt.IsGeneric, $"Class name: {subjectClass.Name}"); - } + //var expectedIsGeneric = expectedGenericType.IsGeneric ? "be generic" : "not be generic"; + //var actualIsNotGeneric = expectedGenericType.IsGeneric ? "is not" : "is generic"; + //assertion.ForCondition(expectedGenericType.IsGeneric == ((IGenericType)actualMemberType).IsGeneric) + // .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to {expectedIsGeneric}{{reason}}, but it {actualIsNotGeneric}."); // See https://github.com/microsoft/python-language-server/issues/1533 on unittest. //Debug.Assert(subjectClass.Bases.Count == otherClass.Bases.Count); //subjectClass.Bases.Count.Should().BeGreaterOrEqualTo(otherClass.Bases.Count); } - if (string.IsNullOrEmpty(subjectMemberType.Documentation)) { - otherMemberType.Documentation.Should().BeNullOrEmpty($"Type name: {subjectMemberType.Name}."); + if (string.IsNullOrEmpty(expectedMemberType.Documentation)) { + assertion.ForCondition(string.IsNullOrEmpty(actualMemberType.Documentation)) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have no documentation{{reason}}, but it has '{actualMemberType.Documentation}'"); } else { - Debug.Assert(subjectMemberType.Documentation == otherMemberType.Documentation); - subjectMemberType.Documentation.Should().Be(otherMemberType.Documentation, $"Type name: {subjectMemberType.Name}."); + assertion.ForCondition(actualMemberType.Documentation.EqualsOrdinal(expectedMemberType.Documentation)) + .FailWith($"Expected python type of '{GetName(subjectType)}.{n}' to have documentation '{expectedMemberType.Documentation}'{{reason}}, but it has '{actualMemberType.Documentation}'"); } - switch (subjectMemberType.MemberType) { + switch (actualMemberType.MemberType) { case PythonMemberType.Class: // Restored collections (like instance of tuple) turn into classes // rather than into collections with content since we don't track @@ -189,20 +193,20 @@ public void HaveSameMembersAs(IMember other) { break; case PythonMemberType.Function: case PythonMemberType.Method: - subjectMemberType.Should().BeAssignableTo(); - otherMemberType.Should().BeAssignableTo(); - if (subjectMemberType is IPythonFunctionType subjectFunction) { - var otherFunction = (IPythonFunctionType)otherMemberType; + actualMemberType.Should().BeAssignableTo(); + expectedMemberType.Should().BeAssignableTo(); + if (actualMemberType is IPythonFunctionType subjectFunction) { + var otherFunction = (IPythonFunctionType)expectedMemberType; subjectFunction.Should().HaveSameOverloadsAs(otherFunction); } break; case PythonMemberType.Property: - subjectMemberType.Should().BeAssignableTo(); - otherMemberType.Should().BeAssignableTo(); + actualMemberType.Should().BeAssignableTo(); + expectedMemberType.Should().BeAssignableTo(); break; case PythonMemberType.Unknown: - subjectMemberType.IsUnknown().Should().BeTrue(); + actualMemberType.IsUnknown().Should().BeTrue(); break; } } diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index 62cbea591..32a5b32d9 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -24,7 +24,6 @@ using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -320,7 +319,8 @@ public async Task DeepSubmoduleImport() { await CreateServicesAsync(PythonVersions.LatestAvailable3X); var rdt = Services.GetService(); - var appDoc = rdt.OpenDocument(appUri, "import top.sub1.sub2.sub3.sub4"); + var appDoc = rdt.OpenDocument(appUri, @"import top.sub1.sub2.sub3.sub4 +x = top.sub1.sub2.sub3.sub4.f"); await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await appDoc.GetAnalysisAsync(Timeout.Infinite); diff --git a/src/Analysis/Ast/Test/LibraryTests.cs b/src/Analysis/Ast/Test/LibraryTests.cs index 721fd79d3..c34163c03 100644 --- a/src/Analysis/Ast/Test/LibraryTests.cs +++ b/src/Analysis/Ast/Test/LibraryTests.cs @@ -80,8 +80,8 @@ import requests Assert.Inconclusive("'requests' package is not installed."); } - var r = analysis.Should().HaveVariable("x").OfType("Response").Which; - r.Should().HaveMember("encoding").Which.Should().HaveType(BuiltinTypeId.Str); + var r = analysis.Should().HaveVariable("x").OfType("Response") + .Which.Should().HaveMember("encoding").Which.Should().HaveType(BuiltinTypeId.Str); } [TestMethod, Priority(0)] diff --git a/src/Analysis/Ast/Test/LocationLoopResolverTests.cs b/src/Analysis/Ast/Test/LocationLoopResolverTests.cs new file mode 100644 index 000000000..acf2a50db --- /dev/null +++ b/src/Analysis/Ast/Test/LocationLoopResolverTests.cs @@ -0,0 +1,65 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using FluentAssertions; +using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.UnitTests.Core.MSTest; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class LocationLoopResolverTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + // ReSharper disable StringLiteralTypo + [PermutationDataRow("A1B1", "B0C1", "C0A0")] + [PermutationDataRow("A2B8", "B2A0", "B6C0", "C3B4")] + // ReSharper restore StringLiteralTypo + [DataTestMethod] + public void FindStartingItem(params string[] input) { + var edges = input.Select(s => (s[0], (int)s[1] - 0x30, s[2], (int)s[3] - 0x30)); + LocationLoopResolver.FindStartingItems(edges).Should().Equal('A'); + } + + // ReSharper disable StringLiteralTypo + [PermutationDataRow("A0B1", "B0A1")] + [PermutationDataRow("A0B1", "B0C1", "C0A1")] + // ReSharper restore StringLiteralTypo + [DataTestMethod] + public void NoStartingItem(params string[] input) { + var edges = input.Select(s => (s[0], (int)s[1] - 0x30, s[2], (int)s[3] - 0x30)); + LocationLoopResolver.FindStartingItems(edges).Should().BeEmpty(); + } + + // ReSharper disable StringLiteralTypo + [PermutationDataRow("A2B4", "B2A0", "C3B4")] + [PermutationDataRow("A2B4", "B2A0", "C2D4", "D2C0")] + // ReSharper restore StringLiteralTypo + [DataTestMethod] + public void TwoStartingItems(params string[] input) { + var edges = input.Select(s => (s[0], (int)s[1] - 0x30, s[2], (int)s[3] - 0x30)); + LocationLoopResolver.FindStartingItems(edges).Should().BeEquivalentTo('A', 'C'); + } + } +} diff --git a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj index 0797b6812..a120fd64b 100644 --- a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj +++ b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj @@ -47,4 +47,4 @@ - + \ No newline at end of file diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index e630d801f..b922e576b 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -45,10 +45,13 @@ public static TypeVarModel FromGeneric(IVariable v) { }; } - public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) - => new GenericTypeParameter(Name, mf.Module, - Constraints.Select(mf.ConstructType).ToArray(), - mf.ConstructType(Bound), Covariant, Contravariant, default); + public override IMember Create(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { + var bound = mf.ConstructType(Bound); + bound = bound.IsUnknown() ? null : bound; + return new GenericTypeParameter(Name, mf.Module, + Constraints.Select(mf.ConstructType).ToArray(), + bound, Covariant, Contravariant, default); + } public override void Populate(ModuleFactory mf, IPythonType declaringType, IGlobalScope gs) { } } diff --git a/src/Caching/Impl/ModuleFactory.cs b/src/Caching/Impl/ModuleFactory.cs index c9794a714..8821768ca 100644 --- a/src/Caching/Impl/ModuleFactory.cs +++ b/src/Caching/Impl/ModuleFactory.cs @@ -50,7 +50,7 @@ public ModuleFactory(ModuleModel model, IPythonModule module, IGlobalScope gs) { } public IPythonType ConstructType(string qualifiedName) - => ConstructMember(qualifiedName)?.GetPythonType(); + => ConstructMember(qualifiedName)?.GetPythonType() ?? Module.Interpreter.UnknownType; public IMember ConstructMember(string qualifiedName) { // Determine module name, member chain and if this is an instance. @@ -96,7 +96,7 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { } if (memberName == "") { - return null; + return new PythonFunctionType("", default, default, string.Empty); } var nextModel = currentModel.GetModel(memberName); @@ -114,11 +114,8 @@ private IMember GetMemberFromThisModule(IReadOnlyList memberNames) { } currentModel = nextModel; - declaringType = m as IPythonType; + declaringType = m.GetPythonType(); Debug.Assert(declaringType != null); - if (declaringType == null) { - return null; - } } return m; diff --git a/src/Caching/Impl/RestoredGlobalScope.cs b/src/Caching/Impl/RestoredGlobalScope.cs index 77e3bb255..32ac257fa 100644 --- a/src/Caching/Impl/RestoredGlobalScope.cs +++ b/src/Caching/Impl/RestoredGlobalScope.cs @@ -69,6 +69,7 @@ private void DeclareVariables() { #region IScope public string Name { get; } + public PythonAst Ast => null; public ScopeStatement Node => null; public IScope OuterScope => null; public IReadOnlyList Children => Array.Empty(); diff --git a/src/Caching/Test/AnalysisCachingTestBase.cs b/src/Caching/Test/AnalysisCachingTestBase.cs index 8baa49828..9bd42c13d 100644 --- a/src/Caching/Test/AnalysisCachingTestBase.cs +++ b/src/Caching/Test/AnalysisCachingTestBase.cs @@ -58,6 +58,7 @@ protected string GetBaselineFileName(string testName, string suffix = null) internal PythonDbModule CreateDbModule(ModuleModel model, string modulePath) { var dbModule = new PythonDbModule(model, modulePath, Services); + Services.GetService().InvalidateAnalysis(dbModule); dbModule.Construct(model); return dbModule; } diff --git a/src/Caching/Test/FluentAssertions/AssertionsFactory.cs b/src/Caching/Test/FluentAssertions/AssertionsFactory.cs index 594897a40..fc59d467b 100644 --- a/src/Caching/Test/FluentAssertions/AssertionsFactory.cs +++ b/src/Caching/Test/FluentAssertions/AssertionsFactory.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System.Diagnostics.CodeAnalysis; +using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; diff --git a/src/Core/Impl/Extensions/TaskExtensions.cs b/src/Core/Impl/Extensions/TaskExtensions.cs index e8e3084a5..e362c44ed 100644 --- a/src/Core/Impl/Extensions/TaskExtensions.cs +++ b/src/Core/Impl/Extensions/TaskExtensions.cs @@ -18,6 +18,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Core.Testing; +using Microsoft.Python.Core.Threading; namespace Microsoft.Python.Core { public static class TaskExtensions { @@ -110,14 +111,17 @@ private static void DoNotWaitSynchronizationContextContinuation(Task task, objec /// /// Attach new to the given task. - /// - /// this allows caller to have its own cancellation without aborting underlying work. - /// - /// if uses different cancellation token than one given - /// it will throw instead of and - /// Task will be set to faulted rather than cancelled. + /// This allows caller to have its own cancellation without aborting underlying work. /// - public static Task WaitAsync(this Task task, CancellationToken cancellationToken) - => task.ContinueWith(t => t.WaitAndUnwrapExceptions(), cancellationToken, TaskContinuationOptions.None, TaskScheduler.Default); + public static Task WaitAsync(this Task task, CancellationToken cancellationToken) { + if (task.IsCompleted || !cancellationToken.CanBeCanceled) { + return task; + } + + var tcs = new TaskCompletionSource(); + tcs.RegisterForCancellation(cancellationToken).UnregisterOnCompletion(task); + task.SetCompletionResultTo(tcs); + return tcs.Task; + } } } diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index abccd0497..ff77b121d 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -602,13 +602,15 @@ from os import path as os_path reference = ds.FindDefinition(analysis, new SourceLocation(4, 12), out _); reference.Should().NotBeNull(); - line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; + var osPyPath = reference.uri.AbsolutePath; + line = File.ReadAllLines(osPyPath)[reference.range.start.line]; line.Should().EndWith("as path"); line.Substring(reference.range.start.character).Should().Be("path"); reference = ds.FindDefinition(analysis, new SourceLocation(5, 12), out _); reference.Should().NotBeNull(); - line = File.ReadAllLines(reference.uri.AbsolutePath)[reference.range.start.line]; + reference.uri.AbsolutePath.Should().Be(osPyPath); + line = File.ReadAllLines(osPyPath)[reference.range.start.line]; line.Should().EndWith("as path"); line.Substring(reference.range.start.character).Should().Be("path"); } diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index b114e05ff..516e0f062 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -475,6 +475,129 @@ from module3 import A3 comps.Should().HaveLabels("M1"); } + [TestMethod, Priority(0)] + public async Task LoopImports_Variables1() { + const string module1Code = @" +class A1: + def M1(self): return 0; pass + +from module2 import y3 +x = y3.M3() +"; + const string module2Code = @" +from module1 import A1 +y1 = A1() +from module3 import A3 +y3 = A3() +"; + const string module3Code = @" +class A3: + def M3(self): return '0'; pass + +from module2 import y1 +z = y1.M1() +"; + + const string appCode = @" +from module1 import x +from module3 import z + +x. +z."; + var module1Uri = TestData.GetTestSpecificUri("module1.py"); + var module2Uri = TestData.GetTestSpecificUri("module2.py"); + var module3Uri = TestData.GetTestSpecificUri("module3.py"); + var appUri = TestData.GetTestSpecificUri("app.py"); + + var root = Path.GetDirectoryName(appUri.AbsolutePath); + await CreateServicesAsync(root, PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var analyzer = Services.GetService(); + + rdt.OpenDocument(module3Uri, module3Code); + rdt.OpenDocument(module2Uri, module2Code); + rdt.OpenDocument(module1Uri, module1Code); + + var app = rdt.OpenDocument(appUri, appCode); + await analyzer.WaitForCompleteAnalysisAsync(); + var analysis = await app.GetAnalysisAsync(-1); + + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); + var comps = cs.GetCompletions(analysis, new SourceLocation(5, 3)); + comps.Should().HaveLabels("capitalize"); + + comps = cs.GetCompletions(analysis, new SourceLocation(6, 3)); + comps.Should().HaveLabels("bit_length"); + } + + [TestMethod, Priority(0)] + public async Task LoopImports_Variables2() { + const string module1Code = @" +from module3 import A3 + +class A1: + def M1(self) -> A3: pass + +from module2 import y3 +x = y3.M3() +"; + const string module2Code = @" +from module1 import A1 +y1 = A1() +from module3 import A3 +y3 = A3() +"; + const string module3Code = @" +from module1 import A1 + +class A3: + def M3(self) -> A1: pass + +from module2 import y1 +z = y1.M1() +"; + const string appCode = @" +from module1 import x +from module3 import z + +x. +z. + +x.M1(). +z.M3(). +"; + var module1Uri = TestData.GetTestSpecificUri("module1.py"); + var module2Uri = TestData.GetTestSpecificUri("module2.py"); + var module3Uri = TestData.GetTestSpecificUri("module3.py"); + var appUri = TestData.GetTestSpecificUri("app.py"); + + var root = Path.GetDirectoryName(appUri.AbsolutePath); + await CreateServicesAsync(root, PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var analyzer = Services.GetService(); + + rdt.OpenDocument(module3Uri, module3Code); + rdt.OpenDocument(module2Uri, module2Code); + rdt.OpenDocument(module1Uri, module1Code); + + var app = rdt.OpenDocument(appUri, appCode); + await analyzer.WaitForCompleteAnalysisAsync(); + var analysis = await app.GetAnalysisAsync(-1); + + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion, Services); + var comps = cs.GetCompletions(analysis, new SourceLocation(5, 3)); + comps.Should().HaveLabels("M1"); + + comps = cs.GetCompletions(analysis, new SourceLocation(6, 3)); + comps.Should().HaveLabels("M3"); + + comps = cs.GetCompletions(analysis, new SourceLocation(8, 8)); + comps.Should().HaveLabels("M3"); + + comps = cs.GetCompletions(analysis, new SourceLocation(9, 8)); + comps.Should().HaveLabels("M1"); + } + [TestMethod, Priority(0)] public async Task TypingModule() { var analysis = await GetAnalysisAsync(@"from typing import "); diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 5ebeee6e7..58c99e8c4 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -49,12 +49,14 @@ public PythonAst(IEnumerable existingAst) { locs.AddRange(a.NewLineLocations.Select(ll => new NewLineLocation(ll.EndIndex + offset, ll.Kind))); offset = locs.LastOrDefault().EndIndex; } + NewLineLocations = locs.ToArray(); offset = 0; foreach (var a in asts) { comments.AddRange(a.CommentLocations.Select(cl => new SourceLocation(cl.Line + offset, cl.Column))); offset += a.NewLineLocations.Length + 1; } + CommentLocations = comments.ToArray(); } @@ -74,12 +76,13 @@ public PythonAst(IEnumerable existingAst) { /// public bool HasVerbatim { get; internal set; } - public override IEnumerable GetChildNodes() => new[] { _body }; + public override IEnumerable GetChildNodes() => new[] {_body}; public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { _body.Walk(walker); } + walker.PostWalk(this); } @@ -87,15 +90,16 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken if (await walker.WalkAsync(this, cancellationToken)) { await _body.WalkAsync(walker, cancellationToken); } + await walker.PostWalkAsync(this, cancellationToken); } public override Statement Body => _body; public PythonLanguageVersion LanguageVersion { get; } - public void Reduce(Func filter) { + public void ReduceToImports() { lock (_lock) { - (Body as SuiteStatement)?.FilterStatements(filter); + (Body as SuiteStatement)?.ReduceToImports(); _attributes?.Clear(); Variables?.Clear(); CommentLocations = Array.Empty(); @@ -121,6 +125,7 @@ public void SetAttribute(Node node, object key, object value) { if (!_attributes.TryGetValue(node, out var nodeAttrs)) { nodeAttrs = _attributes[node] = new Dictionary(); } + nodeAttrs[key] = value; } } @@ -141,8 +146,10 @@ internal void SetAttributes(Dictionary> attribu } #region ILocationConverter + public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index); public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex); + #endregion internal int GetLineEndFromPosition(int index) { @@ -150,6 +157,7 @@ internal int GetLineEndFromPosition(int index) { if (loc.Line >= NewLineLocations.Length) { return index; } + var res = NewLineLocations[loc.Line - 1]; switch (res.Kind) { case NewLineKind.LineFeed: @@ -164,8 +172,7 @@ internal int GetLineEndFromPosition(int index) { internal override bool ExposesLocalVariable(PythonVariable variable) => true; - internal override void FinishBind(PythonNameBinder binder) { - } + internal override void FinishBind(PythonNameBinder binder) { } internal override PythonVariable BindReference(PythonNameBinder binder, string name) => EnsureVariable(name); @@ -186,6 +193,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow return true; } } + variable = null; return false; } @@ -197,7 +205,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow /// for variables explicitly declared global by the user, and names accessed /// but not defined in the lexical scope. /// - internal PythonVariable/*!*/ EnsureGlobalVariable(string name) { + internal PythonVariable /*!*/ EnsureGlobalVariable(string name) { if (!TryGetVariable(name, out var variable)) { variable = CreateVariable(name, VariableKind.Global); } @@ -206,7 +214,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow } - internal PythonVariable/*!*/ EnsureNonlocalVariable(string name) { + internal PythonVariable /*!*/ EnsureNonlocalVariable(string name) { if (!TryGetVariable(name, out var variable)) { variable = CreateVariable(name, VariableKind.Nonlocal); } diff --git a/src/Parsing/Impl/Ast/SuiteStatement.cs b/src/Parsing/Impl/Ast/SuiteStatement.cs index 920a1d8b6..25fd90d72 100644 --- a/src/Parsing/Impl/Ast/SuiteStatement.cs +++ b/src/Parsing/Impl/Ast/SuiteStatement.cs @@ -14,9 +14,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; -using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -33,8 +31,8 @@ public SuiteStatement(Statement[] statements) { public IList Statements => _statements; public override IEnumerable GetChildNodes() => _statements.WhereNotNull(); - public void FilterStatements(Func filter) - => _statements = _statements.Where(filter).ToArray(); + public void ReduceToImports() + => _statements = new FilteredWalker(this).Statements.ToArray(); public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { @@ -164,5 +162,22 @@ public override void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace) { _statements[0].SetLeadingWhiteSpace(ast, whiteSpace); } } + + private sealed class FilteredWalker : PythonWalker { + public FilteredWalker(Node n) { + n.Walk(this); + } + + public List Statements { get; } = new List(); + + public override bool Walk(ImportStatement s) { + Statements.Add(s); + return false; + } + public override bool Walk(FromImportStatement s) { + Statements.Add(s); + return false; + } + } } } diff --git a/src/UnitTests/Core/Impl/MSTest/PermutationalTestMethodAttribute.cs b/src/UnitTests/Core/Impl/MSTest/PermutationDataRowAttribute.cs similarity index 60% rename from src/UnitTests/Core/Impl/MSTest/PermutationalTestMethodAttribute.cs rename to src/UnitTests/Core/Impl/MSTest/PermutationDataRowAttribute.cs index 524c7da90..c78f64454 100644 --- a/src/UnitTests/Core/Impl/MSTest/PermutationalTestMethodAttribute.cs +++ b/src/UnitTests/Core/Impl/MSTest/PermutationDataRowAttribute.cs @@ -1,4 +1,4 @@ -// Python Tools for Visual Studio +// Python Tools for Visual Studio // Copyright(c) Microsoft Corporation // All rights reserved. // @@ -21,34 +21,46 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Microsoft.Python.UnitTests.Core.MSTest { - [AttributeUsage(AttributeTargets.Method)] - public class PermutationalTestMethodAttribute : DataTestMethodAttribute, ITestDataSource { - private readonly int _count; - private readonly int[] _fixedPermutation; + [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] + public class PermutationDataRowAttribute : Attribute, ITestDataSource { + private readonly object[] _source; - public string NameFormat { get; set; } = "{0} ({1})"; + public PermutationDataRowAttribute() { + _source = new object[0]; + } - public PermutationalTestMethodAttribute(int count, params int[] fixedPermutation) { - _count = count; - _fixedPermutation = fixedPermutation; + public PermutationDataRowAttribute(object data) { + _source = new object[1] {data}; } - public IEnumerable GetData(MethodInfo methodInfo) { - if (_fixedPermutation != null && _fixedPermutation.Length > 0) { - yield return new object[] { _fixedPermutation }; - yield break; + public PermutationDataRowAttribute(object data, params object[] moreData) { + if (moreData == null) { + moreData = new object[1]; } - var permutationsIndexes = GetPermutationIndexes(_count); - foreach (var permutationIndexes in permutationsIndexes) { - yield return new object []{ permutationIndexes }; - } + _source = new object[moreData.Length + 1]; + _source[0] = data; + Array.Copy(moreData, 0, _source, 1, moreData.Length); } - public string GetDisplayName(MethodInfo methodInfo, object[] data) { - var names = string.Join(", ", (int[])data[0]); - return string.Format(CultureInfo.InvariantCulture, NameFormat, methodInfo.Name, names); + public IEnumerable GetData(MethodInfo methodInfo) { + var permutationsIndexes = GetPermutationIndexes(_source.Length); + var data = new object[permutationsIndexes.Length][]; + for (var dataIndex = 0; dataIndex < permutationsIndexes.Length; dataIndex++) { + var permutationIndexes = permutationsIndexes[dataIndex]; + var dataRow = new object[_source.Length]; + for (var i = 0; i < dataRow.Length; i++) { + dataRow[i] = _source[permutationIndexes[i]]; + } + + data[dataIndex] = dataRow; + } + + return data; } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + => data == null ? null : string.Format(CultureInfo.InvariantCulture, "{0} ({1})", methodInfo.Name, string.Join(", ", data)); private int[][] GetPermutationIndexes(int count) { if (count == 0) { diff --git a/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs b/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs index 7f94fb7e5..3a34a8ba0 100644 --- a/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs +++ b/src/UnitTests/Core/Impl/TestEnvironmentImpl.cs @@ -28,16 +28,22 @@ namespace TestUtilities { public class TestEnvironmentImpl { private static readonly FieldInfo _stackTraceStringField = typeof(Exception).GetField("_stackTraceString", BindingFlags.Instance | BindingFlags.NonPublic); private static readonly FieldInfo _showDialogField = typeof(Debug).GetField("s_ShowDialog", BindingFlags.Static | BindingFlags.NonPublic); + private static readonly FieldInfo _debugProviderField = typeof(Debug).GetField("s_provider", BindingFlags.Static | BindingFlags.NonPublic); protected internal static TestEnvironmentImpl Instance { get; protected set; } protected TestEnvironmentImpl() { - TryOverrideShowDialog(); + TryOverrideDebugFail(); } - private static void TryOverrideShowDialog() { + private static void TryOverrideDebugFail() { if (_showDialogField != null) { _showDialogField.SetValue(null, new Action(ThrowAssertException)); + } else if (_debugProviderField != null) { + var failCoreField = _debugProviderField.FieldType.GetField("s_FailCore", BindingFlags.Static | BindingFlags.NonPublic); + if (failCoreField != null) { + failCoreField.SetValue(null, new Action(ThrowAssertException)); + } } }