Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Commit 61d60c4

Browse files
author
Mikhail Arkhipov
authored
Persistent analysis, part I (#1224)
* Remove old qualified name * Node storage * Class and scope to use AST map * Library analysis * Fix SO * Keep small AST with imports * AST reduction * Final field * Initial * Reload * Ignore post-final requests * Drop AST * Remove local variables * Test fixes * Fix overload match * Tests * Add locks * Remove local variables * Drop file content to save memory * Cache PEP hints * Recreate AST * Fix specialization * Fix locations * usings * Test fixes * Add options to keep data in memory * Fix test * Fix lambda parameters * Fix argument set Fix global scope node * Fix overload doc * Fix stub merge errors * Fix async issues * Undo some changes * Fix test * Fix race condition * Partial * Models and views * Restore log null checks * Fix merge conflict * Fix merge issue * Null check * Partial * Partial * Partial * Fix test * Partial * Partial * First test * Baseline comparison * Builtins * Partial * Type fixes * Fix type names, part I * Qualified name * Properly write variables * Partial * Construct module from model * Test * Variable creations * Factories * Factories * Split construction * Restore * Save builtins * Test passes * Qualified name * Better export detection * Test fixes * More consistent qualified names * Sys test * Demo * Complete sys write/read * Partial * Partial * Test staility * Perf bug * Baseline, remove debug code, deactivate db * Test fixes * Test fix * Simplify a bit * Baselines and use : separator * Baselines * PR feedback * Merge master * Remove registry reference * PR feedback
1 parent 92ce32a commit 61d60c4

File tree

106 files changed

+42100
-544
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

106 files changed

+42100
-544
lines changed

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
using System.Diagnostics;
1818
using System.Linq;
1919
using Microsoft.Python.Analysis.Documents;
20-
using Microsoft.Python.Analysis.Extensions;
2120
using Microsoft.Python.Analysis.Modules;
2221
using Microsoft.Python.Analysis.Types;
2322
using Microsoft.Python.Analysis.Values;

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public IMember GetValueFromList(ListExpression expression) {
6262
var value = GetValueFromExpression(item) ?? UnknownType;
6363
contents.Add(value);
6464
}
65-
return PythonCollectionType.CreateList(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
65+
return PythonCollectionType.CreateList(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
6666
}
6767

6868
public IMember GetValueFromDictionary(DictionaryExpression expression) {
@@ -72,7 +72,7 @@ public IMember GetValueFromDictionary(DictionaryExpression expression) {
7272
var value = GetValueFromExpression(item.SliceStop) ?? UnknownType;
7373
contents[key] = value;
7474
}
75-
return new PythonDictionary(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
75+
return new PythonDictionary(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
7676
}
7777

7878
private IMember GetValueFromTuple(TupleExpression expression) {
@@ -81,7 +81,7 @@ private IMember GetValueFromTuple(TupleExpression expression) {
8181
var value = GetValueFromExpression(item) ?? UnknownType;
8282
contents.Add(value);
8383
}
84-
return PythonCollectionType.CreateTuple(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
84+
return PythonCollectionType.CreateTuple(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
8585
}
8686

8787
public IMember GetValueFromSet(SetExpression expression) {
@@ -90,7 +90,7 @@ public IMember GetValueFromSet(SetExpression expression) {
9090
var value = GetValueFromExpression(item) ?? UnknownType;
9191
contents.Add(value);
9292
}
93-
return PythonCollectionType.CreateSet(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
93+
return PythonCollectionType.CreateSet(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
9494
}
9595

9696
public IMember GetValueFromGenerator(GeneratorExpression expression) {
@@ -110,14 +110,14 @@ public IMember GetValueFromComprehension(Comprehension node) {
110110
switch (node) {
111111
case ListComprehension lc:
112112
var v1 = GetValueFromExpression(lc.Item) ?? UnknownType;
113-
return PythonCollectionType.CreateList(Interpreter, new[] { v1 });
113+
return PythonCollectionType.CreateList(Module, new[] { v1 });
114114
case SetComprehension sc:
115115
var v2 = GetValueFromExpression(sc.Item) ?? UnknownType;
116-
return PythonCollectionType.CreateSet(Interpreter, new[] { v2 });
116+
return PythonCollectionType.CreateSet(Module, new[] { v2 });
117117
case DictionaryComprehension dc:
118118
var k = GetValueFromExpression(dc.Key) ?? UnknownType;
119119
var v = GetValueFromExpression(dc.Value) ?? UnknownType;
120-
return new PythonDictionary(new PythonDictionaryType(Interpreter), new Dictionary<IMember, IMember> { { k, v } });
120+
return new PythonDictionary(new PythonDictionaryType(Interpreter.ModuleResolution.BuiltinsModule), new Dictionary<IMember, IMember> { { k, v } });
121121
}
122122

123123
return UnknownType;

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,9 @@ private IMember GetValueFromBinaryOp(Expression expr) {
106106

107107
switch (leftTypeId) {
108108
case BuiltinTypeId.List:
109-
return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, lc, rc);
109+
return PythonCollectionType.CreateConcatenatedList(Module, lc, rc);
110110
case BuiltinTypeId.Tuple:
111-
return PythonCollectionType.CreateConcatenatedTuple(Module.Interpreter, lc, rc);
111+
return PythonCollectionType.CreateConcatenatedTuple(Module, lc, rc);
112112
}
113113
}
114114

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalSc
5959
public ModuleSymbolTable SymbolTable { get; } = new ModuleSymbolTable();
6060
public IPythonType UnknownType => Interpreter.UnknownType;
6161
public Location DefaultLocation { get; }
62+
public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule;
6263

6364
public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty;
6465

‎src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ private PythonVariableModule GetOrCreateVariableModule(in string fullName, in Py
174174
return variableModule;
175175
}
176176

177-
variableModule = new PythonVariableModule(fullName, Interpreter);
177+
variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
178178
_variableModules[fullName] = variableModule;
179179
parentModule?.AddChildModule(memberName, variableModule);
180180
return variableModule;

‎src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ private void HandleAllAppendExtend(CallExpression node) {
107107
IPythonCollection values = null;
108108
switch (me.Name) {
109109
case "append":
110-
values = PythonCollectionType.CreateList(Module.Interpreter, new List<IMember> { v }, exact: true);
110+
values = PythonCollectionType.CreateList(Module, new List<IMember> { v }, exact: true);
111111
break;
112112
case "extend":
113113
values = v as IPythonCollection;
@@ -129,7 +129,7 @@ private void ExtendAll(Node location, IPythonCollection values) {
129129
}
130130

131131
var all = scope.Variables[AllVariableName]?.Value as IPythonCollection;
132-
var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, all, values);
132+
var list = PythonCollectionType.CreateConcatenatedList(Module, all, values);
133133
var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
134134

135135
Eval.DeclareVariable(AllVariableName, list, source, location);
@@ -228,6 +228,8 @@ private void MergeStub() {
228228
return;
229229
}
230230

231+
var builtins = Module.Interpreter.ModuleResolution.BuiltinsModule;
232+
231233
// Note that scrape can pick up more functions than the stub contains
232234
// Or the stub can have definitions that scraping had missed. Therefore
233235
// merge is the combination of the two with the documentation coming
@@ -258,6 +260,10 @@ private void MergeStub() {
258260

259261
var memberType = member?.GetPythonType();
260262
var stubMemberType = stubMember.GetPythonType();
263+
264+
if (builtins.Equals(memberType?.DeclaringModule) || builtins.Equals(stubMemberType.DeclaringModule)) {
265+
continue; // Leave builtins alone.
266+
}
261267
if (!IsStubBetterType(memberType, stubMemberType)) {
262268
continue;
263269
}
@@ -271,7 +277,7 @@ private void MergeStub() {
271277
// Re-declare variable with the data from the stub unless member is a module.
272278
// Modules members that are modules should remain as they are, i.e. os.path
273279
// should remain library with its own stub attached.
274-
if (!(stubType is IPythonModule)) {
280+
if (!(stubType is IPythonModule) && !builtins.Equals(stubType.DeclaringModule)) {
275281
sourceType.TransferDocumentationAndLocation(stubType);
276282
// TODO: choose best type between the scrape and the stub. Stub probably should always win.
277283
var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration;

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@ public PythonAnalyzer(IServiceManager services, string cacheFolderPath = null) {
5656
_startNextSession = StartNextSession;
5757

5858
_progress = new ProgressReporter(services.GetService<IProgressService>());
59-
_services.AddService(new StubCache(_services, cacheFolderPath));
59+
60+
_services.AddService(new CacheFolderService(_services, cacheFolderPath));
61+
_services.AddService(new StubCache(_services));
6062
}
6163

6264
public void Dispose() {

‎src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using System.Runtime;
2020
using System.Threading;
2121
using System.Threading.Tasks;
22+
using Microsoft.Python.Analysis.Caching;
2223
using Microsoft.Python.Analysis.Dependencies;
2324
using Microsoft.Python.Analysis.Diagnostics;
2425
using Microsoft.Python.Analysis.Modules;

‎src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
using System.Diagnostics;
1919
using System.Linq;
2020
using Microsoft.Python.Analysis.Analyzer.Evaluation;
21-
using Microsoft.Python.Analysis.Extensions;
2221
using Microsoft.Python.Analysis.Modules;
2322
using Microsoft.Python.Analysis.Types;
2423
using Microsoft.Python.Analysis.Values;

‎src/Analysis/Ast/Impl/Caching/CacheFolders.cs

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,26 +19,27 @@
1919
using System.Security.Cryptography;
2020
using System.Text;
2121
using Microsoft.Python.Core;
22-
using Microsoft.Python.Core.IO;
2322
using Microsoft.Python.Core.Logging;
2423
using Microsoft.Python.Core.OS;
2524

2625
namespace Microsoft.Python.Analysis.Caching {
27-
internal static class CacheFolders {
28-
public static string GetCacheFilePath(string root, string moduleName, string content, IFileSystem fs) {
29-
// Folder for all module versions and variants
30-
// {root}/module_name/content_hash.pyi
31-
var folder = Path.Combine(root, moduleName);
32-
33-
var filePath = Path.Combine(root, folder, $"{FileNameFromContent(content)}.pyi");
34-
if (fs.StringComparison == StringComparison.Ordinal) {
35-
filePath = filePath.ToLowerInvariant();
36-
}
26+
internal sealed class CacheFolderService: ICacheFolderService {
27+
public CacheFolderService(IServiceContainer services, string cacheRootFolder) {
28+
CacheFolder = cacheRootFolder ?? GetCacheFolder(services);
29+
}
30+
31+
public string CacheFolder { get; }
3732

38-
return filePath;
33+
public string GetFileNameFromContent(string content) {
34+
// File name depends on the content so we can distinguish between different versions.
35+
using (var hash = SHA256.Create()) {
36+
return Convert
37+
.ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(content)))
38+
.Replace('/', '_').Replace('+', '-');
39+
}
3940
}
4041

41-
public static string GetCacheFolder(IServiceContainer services) {
42+
private static string GetCacheFolder(IServiceContainer services) {
4243
var platform = services.GetService<IOSPlatform>();
4344
var logger = services.GetService<ILogger>();
4445

@@ -95,17 +96,6 @@ public static string GetCacheFolder(IServiceContainer services) {
9596
return cachePath;
9697
}
9798

98-
public static string FileNameFromContent(string content) {
99-
// File name depends on the content so we can distinguish between different versions.
100-
var hash = SHA256.Create();
101-
return Convert
102-
.ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(content)))
103-
.Replace('/', '_').Replace('+', '-');
104-
}
105-
106-
public static string GetAnalysisCacheFilePath(string analysisRootFolder, string moduleName, string content, IFileSystem fs)
107-
=> GetCacheFilePath(analysisRootFolder, moduleName, content, fs);
108-
10999
private static bool CheckPathRooted(string varName, string path, ILogger logger) {
110100
if (!string.IsNullOrWhiteSpace(path) && Path.IsPathRooted(path)) {
111101
return true;

‎src/Analysis/Ast/Impl/Caching/Definitions/ICacheFolderService.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright(c) Microsoft Corporation
2+
// All rights reserved.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the License); you may not use
5+
// this file except in compliance with the License. You may obtain a copy of the
6+
// License at http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
10+
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
11+
// MERCHANTABILITY OR NON-INFRINGEMENT.
12+
//
13+
// See the Apache Version 2.0 License for specific language governing
14+
// permissions and limitations under the License.
15+
16+
namespace Microsoft.Python.Analysis.Caching {
17+
public interface ICacheFolderService {
18+
string CacheFolder { get; }
19+
string GetFileNameFromContent(string content);
20+
}
21+
}

‎src/Analysis/Ast/Impl/Caching/Definitions/IModuleDatabaseService.cs

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Copyright(c) Microsoft Corporation
2+
// All rights reserved.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the License); you may not use
5+
// this file except in compliance with the License. You may obtain a copy of the
6+
// License at http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
10+
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
11+
// MERCHANTABILITY OR NON-INFRINGEMENT.
12+
//
13+
// See the Apache Version 2.0 License for specific language governing
14+
// permissions and limitations under the License.
15+
16+
using System.Threading;
17+
using System.Threading.Tasks;
18+
using Microsoft.Python.Analysis.Types;
19+
20+
namespace Microsoft.Python.Analysis.Caching {
21+
internal interface IModuleDatabaseService {
22+
/// <summary>
23+
/// Creates module representation from module persistent state.
24+
/// </summary>
25+
/// <param name="moduleName">Module name. If the name is not qualified
26+
/// the module will ge resolved against active Python version.</param>
27+
/// <param name="filePath">Module file path.</param>
28+
/// <param name="module">Python module.</param>
29+
/// <returns>Module storage state</returns>
30+
ModuleStorageState TryCreateModule(string moduleName, string filePath, out IPythonModule module);
31+
32+
/// <summary>
33+
/// Writes module data to the database.
34+
/// </summary>
35+
Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default);
36+
}
37+
}

‎src/Analysis/Ast/Impl/Caching/Definitions/ModuleStorageState.cs

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Copyright(c) Microsoft Corporation
2+
// All rights reserved.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the License); you may not use
5+
// this file except in compliance with the License. You may obtain a copy of the
6+
// License at http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
10+
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
11+
// MERCHANTABILITY OR NON-INFRINGEMENT.
12+
//
13+
// See the Apache Version 2.0 License for specific language governing
14+
// permissions and limitations under the License.
15+
16+
namespace Microsoft.Python.Analysis.Caching {
17+
/// <summary>
18+
/// Describes module data stored in a database.
19+
/// </summary>
20+
public enum ModuleStorageState {
21+
/// <summary>
22+
/// Module does not exist in the database.
23+
/// </summary>
24+
DoesNotExist,
25+
26+
/// <summary>
27+
/// Partial data. This means module is still being analyzed
28+
/// and the data on the module members is incomplete.
29+
/// </summary>
30+
Partial,
31+
32+
/// <summary>
33+
/// Modules exist and the analysis is complete.
34+
/// </summary>
35+
Complete,
36+
37+
/// <summary>
38+
/// Storage is corrupted or incompatible.
39+
/// </summary>
40+
Corrupted
41+
}
42+
}

‎src/Analysis/Ast/Impl/Caching/StubCache.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,14 @@ internal sealed class StubCache : IStubCache {
2727

2828
private readonly IFileSystem _fs;
2929
private readonly ILogger _log;
30+
private readonly ICacheFolderService _cfs;
3031

31-
public StubCache(IServiceContainer services, string cacheRootFolder = null) {
32+
public StubCache(IServiceContainer services) {
3233
_fs = services.GetService<IFileSystem>();
3334
_log = services.GetService<ILogger>();
3435

35-
cacheRootFolder = cacheRootFolder ?? CacheFolders.GetCacheFolder(services);
36-
StubCacheFolder = Path.Combine(cacheRootFolder, $"stubs.v{_stubCacheFormatVersion}");
36+
_cfs = services.GetService<ICacheFolderService>();
37+
StubCacheFolder = Path.Combine(_cfs.CacheFolder, $"stubs.v{_stubCacheFormatVersion}");
3738
}
3839

3940
public string StubCacheFolder { get; }
@@ -58,7 +59,7 @@ public string GetCacheFilePath(string filePath) {
5859
dir = dir.ToLowerInvariant();
5960
}
6061

61-
var dirHash = CacheFolders.FileNameFromContent(dir);
62+
var dirHash = _cfs.GetFileNameFromContent(dir);
6263
var stubFile = Path.Combine(StubCacheFolder, Path.Combine(dirHash, name));
6364
return Path.ChangeExtension(stubFile, ".pyi");
6465
}

‎src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616

1717
using System.Collections.Generic;
18+
using System.ComponentModel;
1819
using System.Linq;
1920
using Microsoft.Python.Analysis.Analyzer.Evaluation;
2021
using Microsoft.Python.Analysis.Types;
@@ -57,7 +58,7 @@ internal static void DeclareParametersInScope(this IArgumentSet args, Expression
5758
}
5859

5960
if (args.ListArgument != null && !string.IsNullOrEmpty(args.ListArgument.Name)) {
60-
var type = new PythonCollectionType(null, BuiltinTypeId.List, eval.Interpreter, false);
61+
var type = new PythonCollectionType(null, BuiltinTypeId.List, eval.BuiltinsModule, false);
6162
var list = new PythonCollection(type, args.ListArgument.Values);
6263
eval.DeclareVariable(args.ListArgument.Name, list, VariableSource.Declaration, args.ListArgument.Location);
6364
}

0 commit comments

Comments
 (0)