Skip to content
This repository was archived by the owner on Nov 4, 2024. It is now read-only.

Commit 4310371

Browse files
AlexanderSherAlexanderSher
authored
Multiple analysis fixes (microsoft#1297)
Multiple analysis fixes. - Fixes microsoft#1151: Reliable way to determine final analysis of a module - Fixes microsoft#1201: NRE in SymbolCollector.AddProperty - Fixes microsoft#1228: NRE in TryFindMissingDependencies - Fixes microsoft#1294: Handle valid case for reloading AST - Fixes microsoft#1295: FindReferences doesn't work on package init members - Fixes microsoft#1296: DependencyResolver graph misses intermediate imports - Part of the fix for microsoft#1174: All references are not listed when using ms language server (fixes in analysis itself are also required for this one)
1 parent 7fb0777 commit 4310371

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+729
-376
lines changed

‎src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs

Lines changed: 4 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -19,48 +19,21 @@
1919

2020
namespace Microsoft.Python.Analysis.Analyzer {
2121
internal static class ActivityTracker {
22-
private static readonly Dictionary<string, AnalysisState> _modules = new Dictionary<string, AnalysisState>();
22+
private static readonly HashSet<string> _modules = new HashSet<string>();
2323
private static readonly object _lock = new object();
2424
private static bool _tracking;
2525
private static Stopwatch _sw;
2626

27-
private struct AnalysisState {
28-
public int Count;
29-
public bool IsComplete;
30-
}
31-
3227
public static void OnEnqueueModule(string path) {
3328
if (string.IsNullOrEmpty(path)) {
3429
return;
3530
}
3631

3732
lock (_lock) {
38-
if (!_modules.TryGetValue(path, out var st)) {
39-
_modules[path] = default;
40-
} else {
41-
st.IsComplete = false;
42-
}
43-
}
44-
}
45-
46-
public static void OnModuleAnalysisComplete(string path) {
47-
lock (_lock) {
48-
if (_modules.TryGetValue(path, out var st)) {
49-
st.Count++;
50-
st.IsComplete = true;
51-
}
33+
_modules.Add(path);
5234
}
5335
}
5436

55-
public static bool IsAnalysisComplete {
56-
get {
57-
lock (_lock) {
58-
return _modules.All(m => m.Value.IsComplete);
59-
}
60-
}
61-
}
62-
63-
6437
public static void StartTracking() {
6538
lock (_lock) {
6639
if (!_tracking) {
@@ -71,22 +44,15 @@ public static void StartTracking() {
7144
}
7245
}
7346

74-
public static void EndTracking() {
47+
public static (int modulesCount, double totalMilliseconds) EndTracking() {
7548
lock (_lock) {
7649
if (_tracking) {
7750
_sw?.Stop();
7851
_tracking = false;
7952
}
80-
}
81-
}
8253

83-
public static int ModuleCount {
84-
get {
85-
lock (_lock) {
86-
return _modules.Count;
87-
}
54+
return (_modules.Count, _sw?.Elapsed.TotalMilliseconds ?? 0);
8855
}
8956
}
90-
public static double MillisecondsElapsed => _sw?.Elapsed.TotalMilliseconds ?? 0;
9157
}
9258
}

‎src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,56 @@
1515

1616
using System;
1717
using System.Diagnostics;
18+
using Microsoft.Python.Analysis.Documents;
1819
using Microsoft.Python.Analysis.Modules;
1920
using Microsoft.Python.Analysis.Types;
2021
using Microsoft.Python.Core;
2122

2223
namespace Microsoft.Python.Analysis.Analyzer {
2324
[DebuggerDisplay("{Name} : {FilePath}")]
24-
internal struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
25+
internal readonly struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
26+
private enum KeyType { Default, Typeshed, LibraryAsDocument }
27+
28+
private readonly KeyType _type;
2529
public string Name { get; }
2630
public string FilePath { get; }
27-
public bool IsTypeshed { get; }
31+
public bool IsTypeshed => _type == KeyType.Typeshed;
32+
public bool IsLibraryAsDocument => _type == KeyType.LibraryAsDocument;
2833

2934
public AnalysisModuleKey(IPythonModule module) {
3035
Name = module.Name;
3136
FilePath = module.ModuleType == ModuleType.CompiledBuiltin ? null : module.FilePath;
32-
IsTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
37+
_type = module is StubPythonModule stub && stub.IsTypeshed
38+
? KeyType.Typeshed
39+
: module.ModuleType == ModuleType.Library && module is IDocument document && document.IsOpen
40+
? KeyType.LibraryAsDocument
41+
: KeyType.Default;
3342
}
3443

3544
public AnalysisModuleKey(string name, string filePath, bool isTypeshed) {
3645
Name = name;
3746
FilePath = filePath;
38-
IsTypeshed = isTypeshed;
47+
_type = isTypeshed ? KeyType.Typeshed : KeyType.Default;
48+
}
49+
50+
private AnalysisModuleKey(string name, string filePath, KeyType type) {
51+
Name = name;
52+
FilePath = filePath;
53+
_type = type;
3954
}
4055

56+
public AnalysisModuleKey GetLibraryAsDocumentKey() => new AnalysisModuleKey(Name, FilePath, KeyType.LibraryAsDocument);
57+
4158
public bool Equals(AnalysisModuleKey other)
42-
=> Name.EqualsOrdinal(other.Name) && FilePath.PathEquals(other.FilePath) && IsTypeshed == other.IsTypeshed;
59+
=> Name.EqualsOrdinal(other.Name) && FilePath.PathEquals(other.FilePath) && _type == other._type;
4360

4461
public override bool Equals(object obj) => obj is AnalysisModuleKey other && Equals(other);
4562

4663
public override int GetHashCode() {
4764
unchecked {
4865
var hashCode = (Name != null ? Name.GetHashCode() : 0);
4966
hashCode = (hashCode * 397) ^ (FilePath != null ? FilePath.GetPathHashCode() : 0);
50-
hashCode = (hashCode * 397) ^ IsTypeshed.GetHashCode();
67+
hashCode = (hashCode * 397) ^ _type.GetHashCode();
5168
return hashCode;
5269
}
5370
}

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ internal interface IAnalyzable {
2626
/// <summary>
2727
/// Notifies document that its analysis is now complete.
2828
/// </summary>
29-
void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass);
29+
/// <param name="analysis">Document analysis</param>
30+
void NotifyAnalysisComplete(IDocumentAnalysis analysis);
3031
}
3132
}

‎src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,15 @@
2020
using Microsoft.Python.Analysis.Diagnostics;
2121
using Microsoft.Python.Analysis.Types;
2222
using Microsoft.Python.Core.Collections;
23+
using Microsoft.Python.Parsing.Ast;
2324

2425
namespace Microsoft.Python.Analysis.Analyzer {
2526
public interface IPythonAnalyzer {
2627
Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default);
2728
/// <summary>
2829
/// Schedules module for analysis. Module will be scheduled if version of AST is greater than the one used to get previous analysis
2930
/// </summary>
30-
void EnqueueDocumentForAnalysis(IPythonModule module, int version);
31+
void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version);
3132

3233
/// <summary>
3334
/// Schedules module for analysis for its existing AST, but with new dependencies.

‎src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public EmptyAnalysis(IServiceContainer services, IDocument document) {
3030
Document = document ?? throw new ArgumentNullException(nameof(document));
3131
GlobalScope = new EmptyGlobalScope(document);
3232
Ast = AstUtilities.MakeEmptyAst(document.Uri);
33-
ExpressionEvaluator = new ExpressionEval(services, document);
33+
ExpressionEvaluator = new ExpressionEval(services, document, Ast);
3434
}
3535

3636
public IDocument Document { get; }

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr)
9494
var init = cls.GetMember<IPythonFunctionType>(@"__init__");
9595
if (init != null) {
9696
using (OpenScope(cls.DeclaringModule, cls.ClassDefinition, out _)) {
97-
var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, Module, this);
97+
var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, this);
9898
if (a.Errors.Count > 0) {
9999
// AddDiagnostics(Module.Uri, a.Errors);
100100
}

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public void DeclareVariable(string name, IMember value, VariableSource source)
4141
=> DeclareVariable(name, value, source, default(Location));
4242

4343
public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module)
44-
=> DeclareVariable(name, value, source, new Location(module, default));
44+
=> DeclareVariable(name, value, source, new Location(module));
4545

4646
public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = false)
4747
=> DeclareVariable(name, value, source, GetLocationOfName(location), overwrite);

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,12 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator {
3838
private readonly object _lock = new object();
3939
private readonly List<DiagnosticsEntry> _diagnostics = new List<DiagnosticsEntry>();
4040

41-
public ExpressionEval(IServiceContainer services, IPythonModule module)
42-
: this(services, module, new GlobalScope(module)) { }
43-
44-
public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) {
41+
public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) {
4542
Services = services ?? throw new ArgumentNullException(nameof(services));
4643
Module = module ?? throw new ArgumentNullException(nameof(module));
47-
Ast = module.GetAst();
44+
Ast = ast ?? throw new ArgumentNullException(nameof(ast));
4845

49-
GlobalScope = gs;
46+
GlobalScope = new GlobalScope(module, ast);
5047
CurrentScope = GlobalScope;
5148
DefaultLocation = new Location(module);
5249
//Log = services.GetService<ILogger>();
@@ -60,7 +57,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalSc
6057
public IPythonType UnknownType => Interpreter.UnknownType;
6158
public Location DefaultLocation { get; }
6259

63-
public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty;
60+
public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty;
6461

6562
public Location GetLocationOfName(Node node) {
6663
if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) {
@@ -103,7 +100,7 @@ public Location GetLocationOfName(Node node) {
103100
public IServiceContainer Services { get; }
104101
IScope IExpressionEvaluator.CurrentScope => CurrentScope;
105102
IGlobalScope IExpressionEvaluator.GlobalScope => GlobalScope;
106-
public LocationInfo GetLocation(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty;
103+
public LocationInfo GetLocation(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty;
107104
public IEnumerable<DiagnosticsEntry> Diagnostics => _diagnostics;
108105

109106
public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) {

‎src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,8 @@ private void SpecializeFuture(FromImportStatement node) {
125125

126126
var printNameExpression = node.Names.FirstOrDefault(n => n?.Name == "print_function");
127127
if (printNameExpression != null) {
128-
var fn = new PythonFunctionType("print", new Location(Module, default), null, string.Empty);
129-
var o = new PythonFunctionOverload(fn.Name, new Location(Module, default));
128+
var fn = new PythonFunctionType("print", new Location(Module), null, string.Empty);
129+
var o = new PythonFunctionOverload(fn.Name, new Location(Module));
130130
var parameters = new List<ParameterInfo> {
131131
new ParameterInfo("*values", Interpreter.GetBuiltinType(BuiltinTypeId.Object), ParameterKind.List, null),
132132
new ParameterInfo("sep", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.KeywordOnly, null),

‎src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,19 +56,22 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
5656
// import_module('fob.oar.baz')
5757
var importNames = ImmutableArray<string>.Empty;
5858
var variableModule = default(PythonVariableModule);
59+
var importedNamesCount = 0;
5960
foreach (var nameExpression in moduleImportExpression.Names) {
6061
importNames = importNames.Add(nameExpression.Name);
6162
var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute);
6263
if (!HandleImportSearchResult(imports, variableModule, asNameExpression, moduleImportExpression, out variableModule)) {
63-
return;
64+
break;
6465
}
66+
67+
importedNamesCount++;
6568
}
6669

6770
// "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz')
6871
// "import fob.oar.baz" is handled as fob = import_module('fob')
69-
if (!string.IsNullOrEmpty(asNameExpression?.Name)) {
72+
if (!string.IsNullOrEmpty(asNameExpression?.Name) && importedNamesCount == moduleImportExpression.Names.Count) {
7073
Eval.DeclareVariable(asNameExpression.Name, variableModule, VariableSource.Import, asNameExpression);
71-
} else if (importNames.Count > 0 && !string.IsNullOrEmpty(importNames[0]) && _variableModules.TryGetValue(importNames[0], out variableModule)) {
74+
} else if (importedNamesCount > 0 && !string.IsNullOrEmpty(importNames[0]) && _variableModules.TryGetValue(importNames[0], out variableModule)) {
7275
var firstName = moduleImportExpression.Names[0];
7376
Eval.DeclareVariable(importNames[0], variableModule, VariableSource.Import, firstName);
7477
}
@@ -87,7 +90,7 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV
8790
case RelativeImportBeyondTopLevel importBeyondTopLevel:
8891
var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName);
8992
Eval.ReportDiagnostics(Eval.Module.Uri,
90-
new DiagnosticsEntry(message, location.GetLocation(Eval.Module).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
93+
new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
9194
variableModule = default;
9295
return false;
9396
case ImportNotFound importNotFound:

‎src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,22 +29,15 @@ namespace Microsoft.Python.Analysis.Analyzer {
2929
/// Analysis of a library code.
3030
/// </summary>
3131
internal sealed class LibraryAnalysis : IDocumentAnalysis {
32-
public LibraryAnalysis(IDocument document, int version, IServiceContainer services, GlobalScope globalScope, IReadOnlyList<string> starImportMemberNames) {
32+
public LibraryAnalysis(IDocument document, int version, IGlobalScope globalScope, IExpressionEvaluator eval, IReadOnlyList<string> starImportMemberNames) {
3333
Check.ArgumentNotNull(nameof(document), document);
3434
Check.ArgumentNotNull(nameof(globalScope), globalScope);
3535

3636
Document = document;
3737
Version = version;
3838
GlobalScope = globalScope;
3939

40-
var ast = Document.GetAst();
41-
ast.Reduce(x => x is ImportStatement || x is FromImportStatement);
42-
var c = (IAstNodeContainer)Document;
43-
c.ClearContent();
44-
c.ClearAst();
45-
c.AddAstNode(document, ast);
46-
47-
ExpressionEvaluator = new ExpressionEval(services, document, globalScope);
40+
ExpressionEvaluator = eval;
4841
StarImportMemberNames = starImportMemberNames;
4942
}
5043

‎src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ internal class ModuleWalker : AnalysisWalker {
3737
private int _allReferencesCount;
3838
private bool _allIsUsable = true;
3939

40-
public ModuleWalker(IServiceContainer services, IPythonModule module)
41-
: base(new ExpressionEval(services, module)) {
40+
public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast)
41+
: base(new ExpressionEval(services, module, ast)) {
4242
_stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null;
4343
}
4444

0 commit comments

Comments
 (0)