diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index 7ecf1dee4..5c775f42a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -43,15 +43,18 @@ public void DeclareVariable(string name, IMember value, VariableSource source) public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module) => DeclareVariable(name, value, source, new Location(module)); - public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = false) + public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = true) => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite); - public void DeclareVariable(string name, IMember value, VariableSource source, Location location, bool overwrite = false) { + public void DeclareVariable(string name, IMember value, VariableSource source, Location location, bool overwrite = true) { + var member = GetInScope(name); + if (member != null && !overwrite) { + return; + } if (source == VariableSource.Import && value is IVariable v) { CurrentScope.LinkVariable(name, v, location); return; } - var member = GetInScope(name); if (member != null) { if (!value.IsUnknown()) { CurrentScope.DeclareVariable(name, value, source, location); diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index 42b2f958c..7503d5a06 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -58,18 +58,25 @@ public void HandleAssignment(AssignmentStatement node) { } foreach (var ne in node.Left.OfType()) { + IScope scope; if (Eval.CurrentScope.NonLocals[ne.Name] != null) { - Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal); + Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal); scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne)); continue; } - if (Eval.CurrentScope.Globals[ne.Name] != null) { - Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global); + Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global); scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne)); continue; } + var m = Eval.LookupNameInScopes(ne.Name, out scope); + if(m?.MemberType == PythonMemberType.Class) { + // Ignore assignments to classes: enum.py does Enum = None + // which prevents persistence from restoring proper type from enum:Enum. + continue; + } + var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration; Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, Eval.GetLocationOfName(ne)); } @@ -97,7 +104,7 @@ private void TryHandleClassVariable(AssignmentStatement node, IMember value) { var cls = m.GetPythonType(); if (cls != null) { using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) { - Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLocationOfName(mex), true); + Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLocationOfName(mex)); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index 182f83d5d..50edd0261 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -59,7 +59,7 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor // TODO: warn this is not a good style per // TODO: https://docs.python.org/3/faq/programming.html#what-are-the-best-practices-for-using-import-in-a-module // TODO: warn this is invalid if not in the global scope. - HandleModuleImportStar(variableModule, imports is ImplicitPackageImport); + HandleModuleImportStar(variableModule, imports is ImplicitPackageImport, node.StartIndex); return; } @@ -68,14 +68,16 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor if (!string.IsNullOrEmpty(memberName)) { var nameExpression = asNames[i] ?? names[i]; var variableName = nameExpression?.Name ?? memberName; - var exported = variableModule.Analysis?.GlobalScope.Variables[memberName] ?? variableModule.GetMember(memberName); + var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; + var exported = variable ?? variableModule.GetMember(memberName); var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName); - Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression); + // Do not allow imported variables to override local declarations + Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex)); } } } - private void HandleModuleImportStar(PythonVariableModule variableModule, bool isImplicitPackage) { + private void HandleModuleImportStar(PythonVariableModule variableModule, bool isImplicitPackage, int importPosition) { if (variableModule.Module == Module) { // from self import * won't define any new members return; @@ -100,10 +102,31 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, bool is } var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; - Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import); + // Do not allow imported variables to override local declarations + Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import, Eval.DefaultLocation, CanOverwriteVariable(memberName, importPosition)); } } + private bool CanOverwriteVariable(string name, int importPosition) { + var v = Eval.CurrentScope.Variables[name]; + if(v == null) { + return true; // Variable does not exist + } + // Allow overwrite if import is below the variable. Consider + // x = 1 + // x = 2 + // from A import * # brings another x + // x = 3 + var references = v.References.Where(r => r.DocumentUri == Module.Uri).ToArray(); + if(references.Length == 0) { + // No references to the variable in this file - the variable + // is imported from another module. OK to overwrite. + return true; + } + var firstAssignmentPosition = references.Min(r => r.Span.ToIndexSpan(Ast).Start); + return firstAssignmentPosition < importPosition; + } + private IMember GetValueFromImports(PythonVariableModule parentModule, IImportChildrenSource childrenSource, string memberName) { if (childrenSource == null || !childrenSource.TryGetChildImport(memberName, out var childImport)) { return Interpreter.UnknownType; diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index a2c1759b9..e8aba3904 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -255,7 +255,7 @@ private void MergeStub() { switch (sourceType) { case null: // Nothing in sources, but there is type in the stub. Declare it. - if (v.Source == VariableSource.Declaration) { + if (v.Source == VariableSource.Declaration || v.Source == VariableSource.Generic) { Eval.DeclareVariable(v.Name, v.Value, v.Source); } break; @@ -347,6 +347,10 @@ private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubTyp if (sourceType.MemberType == PythonMemberType.Function && stubType.MemberType == PythonMemberType.Class) { return true; } + // Random replaces method (variable) by a function. + if (sourceType.MemberType == PythonMemberType.Method && stubType.MemberType == PythonMemberType.Function) { + return true; + } return sourceType.MemberType == stubType.MemberType; } diff --git a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs index 36c4eb180..dcf73ec35 100644 --- a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs +++ b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs @@ -14,6 +14,25 @@ // permissions and limitations under the License. namespace Microsoft.Python.Analysis { + public enum AnalysisCachingLevel { + /// + /// No caching of analysis. + /// + None, + + /// + /// Cache analysis results of system (language) modules. + /// Do not cache user-installed modules or site-packages. + /// + System, + + /// + /// Full caching, includes system and library modules. + /// Does not enable caching of user code analysis. + /// + Library + } + public class AnalysisOptions { public bool LintingEnabled { get; set; } @@ -29,5 +48,10 @@ public class AnalysisOptions { /// improve performance when library code has to be re-analyzed. /// public bool KeepLibraryAst { get; set; } + + /// + /// Defines level of caching analysis engine will maintain. + /// + public AnalysisCachingLevel AnalysisCachingLevel { get; set; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs index 9aca942a0..77d631f2d 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericTypeParameter.cs @@ -27,5 +27,8 @@ public interface IGenericTypeParameter: IPythonType, IEquatable /// See 'https://docs.python.org/3/library/typing.html#typing.TypeVar' IReadOnlyList Constraints { get; } + IPythonType Bound { get; } + IPythonType Covariant { get; } + IPythonType Contravariant { get; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs index 0cdc143fb..f523601d1 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs @@ -25,15 +25,33 @@ namespace Microsoft.Python.Analysis.Specializations.Typing.Types { internal sealed class GenericTypeParameter : PythonType, IGenericTypeParameter { - public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnlyList constraints, string documentation, IndexSpan location) - : base(name, new Location(declaringModule), documentation) { + public GenericTypeParameter( + string name, + IPythonModule declaringModule, + IReadOnlyList constraints, + IPythonType bound, + IPythonType covariant, + IPythonType contravariant, + Location location) + : base(name, location, GetDocumentation(name, constraints, bound, covariant, contravariant)) { Constraints = constraints ?? Array.Empty(); + Bound = bound; + Covariant = covariant; + Contravariant = contravariant; } + + #region IGenericTypeParameter public IReadOnlyList Constraints { get; } + public IPythonType Bound { get; } + public IPythonType Covariant { get; } + public IPythonType Contravariant { get; } + #endregion + #region IPythonType public override BuiltinTypeId TypeId => BuiltinTypeId.Type; public override PythonMemberType MemberType => PythonMemberType.Generic; public override bool IsSpecialized => true; + #endregion private static bool TypeVarArgumentsValid(IArgumentSet argSet) { var args = argSet.Arguments; @@ -90,23 +108,29 @@ public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declari // Type constraints may be specified as type name strings. var typeString = (a as IPythonConstant)?.GetString(); return !string.IsNullOrEmpty(typeString) ? argSet.Eval.GetTypeFromString(typeString) : a.GetPythonType(); - }).ToArray() ?? Array.Empty(); + }).ToArray(); + + var bound = args.Where(a => a.Name == "bound").Select(a => a.Value as IPythonType).FirstOrDefault(); + var covariant = args.Where(a => a.Name == "covariant").Select(a => a.Value as IPythonType).FirstOrDefault(); + var contravariant = args.Where(a => a.Name == "contravariant").Select(a => a.Value as IPythonType).FirstOrDefault(); - var documentation = GetDocumentation(args, constraints); - return new GenericTypeParameter(name, declaringModule, constraints, documentation, location); + return new GenericTypeParameter(name, declaringModule, constraints, bound, covariant, contravariant, new Location(declaringModule, location)); } - private static string GetDocumentation(IReadOnlyList args, IReadOnlyList constraints) { - var name = (args[0].Value as IPythonConstant).GetString(); - var keyWordArgs = args.Skip(1) - .Where(x => !x.ValueIsDefault) - .Select(x => $"{x.Name}={(x.Value as IPythonConstant)?.Value}"); + private static string GetDocumentation(string name, IReadOnlyList constraints, IPythonType bound, IPythonType covariant, IPythonType contravariant) { + var constaintStrings = constraints != null ? constraints.Select(c => c.IsUnknown() ? "?" : c.Name) : Enumerable.Empty(); + var boundStrings = bound.IsUnknown() ? Enumerable.Empty() : Enumerable.Repeat($"bound={bound.Name}", 1); + var covariantStrings = covariant.IsUnknown() ? Enumerable.Empty() : Enumerable.Repeat($"covariant={covariant.Name}", 1); + var contravariantStrings = contravariant.IsUnknown() ? Enumerable.Empty() : Enumerable.Repeat($"contravariant={contravariant.Name}", 1); + + var docArgs = Enumerable.Repeat($"'{name}'", 1) + .Concat(constaintStrings).Concat(boundStrings).Concat(covariantStrings).Concat(contravariantStrings); - var docArgs = constraints.Select(c => c.IsUnknown() ? "?" : c.Name).Concat(keyWordArgs).Prepend($"'{name}'"); var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs); return documentation; } - public bool Equals(IGenericTypeParameter other) => Name.Equals(other.Name); + public bool Equals(IGenericTypeParameter other) => Name.Equals(other?.Name); } + } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index cc36d4c0d..bd444a1cd 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -103,6 +103,8 @@ private void SpecializeMembers() { _members["ValuesView"] = new SpecializedGenericType("ValuesView", CreateValuesViewType, this); _members["ItemsView"] = new SpecializedGenericType("ItemsView", CreateItemsViewType, this); + _members["AbstractSet"] = new SpecializedGenericType("AbstractSet", + typeArgs => CreateListType("AbstractSet", BuiltinTypeId.Set, typeArgs, true), this); _members["Set"] = new SpecializedGenericType("Set", typeArgs => CreateListType("Set", BuiltinTypeId.Set, typeArgs, true), this); _members["MutableSet"] = new SpecializedGenericType("MutableSet", diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs index fdf4c2188..a4101de0a 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.Generics.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Utilities; @@ -23,13 +24,14 @@ namespace Microsoft.Python.Analysis.Types { internal partial class PythonClassType { + private readonly object _genericParameterLock = new object(); + private readonly ReentrancyGuard _genericSpecializationGuard = new ReentrancyGuard(); + private readonly ReentrancyGuard _genericResolutionGuard = new ReentrancyGuard(); + private bool _isGeneric; - private object _genericParameterLock = new object(); private Dictionary _specificTypeCache; private Dictionary _genericParameters; private IReadOnlyList _parameters = new List(); - private ReentrancyGuard _genericSpecializationGuard = new ReentrancyGuard(); - private ReentrancyGuard _genericResolutionGuard = new ReentrancyGuard(); #region IGenericType /// @@ -66,7 +68,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { // type parameter T -> int, U -> str, etc. var genericTypeToSpecificType = GetSpecificTypes(args, genericTypeParameters, newBases); - PythonClassType classType = new PythonClassType(BaseName, new Location(DeclaringModule)); + var classType = new PythonClassType(BaseName, new Location(DeclaringModule)); // Storing generic parameters allows methods returning generic types // to know what type parameter returns what specific type StoreGenericParameters(classType, genericTypeParameters, genericTypeToSpecificType); diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index b1ac669fe..8a6acfd6c 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -40,7 +40,7 @@ internal enum ClassDocumentationSource { private readonly ReentrancyGuard _memberGuard = new ReentrancyGuard(); private string _genericName; - private List _bases; + private List _bases = new List(); private IReadOnlyList _mro; private string _documentation; @@ -187,7 +187,7 @@ public IReadOnlyList Mro { if (_mro != null) { return _mro; } - if (_bases == null) { + if (_bases.Count == 0) { return new IPythonType[] { this }; } _mro = new IPythonType[] { this }; @@ -211,7 +211,7 @@ public IReadOnlyList Mro { internal override void SetDocumentation(string documentation) => _documentation = documentation; internal void SetBases(IEnumerable bases) { - if (_bases != null) { + if (_bases.Count > 0) { return; // Already set } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index fa6520372..c6d1067b6 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -64,13 +64,12 @@ public override void AddReference(Location location) { #region IPythonType - public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : BaseName; - public virtual string QualifiedName => DeclaringModule.ModuleType == ModuleType.Builtins ? TypeId == BuiltinTypeId.Ellipsis ? "ellipsis" : Name : this.GetQualifiedName(); + public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : BaseName; public virtual string Documentation { get; private set; } public virtual BuiltinTypeId TypeId => _typeId; public string BaseName { get; } diff --git a/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs b/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs index cbccd3e6b..8adbd2782 100644 --- a/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs +++ b/src/Analysis/Ast/Impl/Utilities/ReentrancyGuard.cs @@ -40,7 +40,7 @@ public IDisposable Push(T t, out bool reentered) { public void Pop() { _stack.Value.Pop(); - if(_stack.Value.Count == 0) { + if (_stack.Value.Count == 0) { _stack.Value = null; } } diff --git a/src/Caching/Impl/Extensions/IndexSpanExtensions.cs b/src/Caching/Impl/Extensions/IndexSpanExtensions.cs index 44f43d018..3381df3f7 100644 --- a/src/Caching/Impl/Extensions/IndexSpanExtensions.cs +++ b/src/Caching/Impl/Extensions/IndexSpanExtensions.cs @@ -14,11 +14,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Diagnostics; using Microsoft.Python.Analysis.Caching.Models; using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Caching { internal static class IndexSpanExtensions { + [DebuggerStepThrough] public static IndexSpanModel ToModel(this IndexSpan span) => new IndexSpanModel { Start = span.Start, Length = span.Length diff --git a/src/Caching/Impl/Factories/ModuleFactory.cs b/src/Caching/Impl/Factories/ModuleFactory.cs index 9627076a8..ddc18af8e 100644 --- a/src/Caching/Impl/Factories/ModuleFactory.cs +++ b/src/Caching/Impl/Factories/ModuleFactory.cs @@ -37,6 +37,7 @@ internal sealed class ModuleFactory : IDisposable { public FunctionFactory FunctionFactory { get; } public PropertyFactory PropertyFactory { get; } public VariableFactory VariableFactory { get; } + public TypeVarFactory TypeVarFactory { get; } public Location DefaultLocation { get; } public ModuleFactory(ModuleModel model, IPythonModule module) { @@ -44,6 +45,7 @@ public ModuleFactory(ModuleModel model, IPythonModule module) { ClassFactory = new ClassFactory(model.Classes, this); FunctionFactory = new FunctionFactory(model.Functions, this); VariableFactory = new VariableFactory(model.Variables, this); + TypeVarFactory = new TypeVarFactory(model.TypeVars, this); PropertyFactory = new PropertyFactory(this); DefaultLocation = new Location(Module); } @@ -52,6 +54,7 @@ public void Dispose() { ClassFactory.Dispose(); FunctionFactory.Dispose(); VariableFactory.Dispose(); + TypeVarFactory.Dispose(); } public IPythonType ConstructType(string qualifiedName) => ConstructMember(qualifiedName)?.GetPythonType(); @@ -194,12 +197,14 @@ private IReadOnlyList GetTypeArguments(string memberName, out strin var argumentString = memberName.Substring(openBracket + 1, closeBracket - openBracket - 1); // Extract type names from argument string. Note that types themselves // can have arguments: Union[int, Union[int, Union[str, bool]], ...]. - var arguments = TypeNames.GetTypeNames(argumentString, ','); - foreach (var a in arguments) { - var t = ConstructType(a); - // TODO: better handle generics type definitions from TypeVar. - // https://github.com/microsoft/python-language-server/issues/1214 - t = t ?? new GenericTypeParameter(a, Module, Array.Empty(), string.Empty, DefaultLocation.IndexSpan); + var qualifiedNames = TypeNames.GetTypeNames(argumentString, ','); + foreach (var qn in qualifiedNames) { + var t = ConstructType(qn); + if (t == null) { + TypeNames.DeconstructQualifiedName(qn, out var parts); + typeName = string.Join(".", parts.MemberNames); + t = new GenericTypeParameter(typeName, Module, Array.Empty(), null, null, null, DefaultLocation); + } typeArgs.Add(t); } typeName = memberName.Substring(0, openBracket); diff --git a/src/Caching/Impl/Factories/TypeVarFactory.cs b/src/Caching/Impl/Factories/TypeVarFactory.cs new file mode 100644 index 000000000..f9cf8025c --- /dev/null +++ b/src/Caching/Impl/Factories/TypeVarFactory.cs @@ -0,0 +1,36 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Caching.Models; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; + +namespace Microsoft.Python.Analysis.Caching.Factories { + internal sealed class TypeVarFactory : FactoryBase { + public TypeVarFactory(IEnumerable models, ModuleFactory mf) + : base(models, mf) { + } + + public override IPythonType CreateMember(TypeVarModel tvm, IPythonType declaringType) { + var constraints = tvm.Constraints.Select(c => ModuleFactory.ConstructType(c)).ToArray(); + var bound = ModuleFactory.ConstructType(tvm.Bound); + var covariant = ModuleFactory.ConstructType(tvm.Covariant); + var contravariant = ModuleFactory.ConstructType(tvm.Contravariant); + return new GenericTypeParameter(tvm.Name, ModuleFactory.Module, constraints, bound, covariant, contravariant, ModuleFactory.DefaultLocation); + } + } +} diff --git a/src/Caching/Impl/GlobalScope.cs b/src/Caching/Impl/GlobalScope.cs index d484c0d66..84d7324fc 100644 --- a/src/Caching/Impl/GlobalScope.cs +++ b/src/Caching/Impl/GlobalScope.cs @@ -32,23 +32,26 @@ public GlobalScope(ModuleModel model, IPythonModule module, IServiceContainer se Name = model.Name; using (var mf = new ModuleFactory(model, module)) { - // TODO: store real location in models + foreach (var tvm in model.TypeVars) { + var t = mf.TypeVarFactory.Construct(tvm); + _scopeVariables.DeclareVariable(tvm.Name, t, VariableSource.Generic, mf.DefaultLocation); + } // Member creation may be non-linear. Consider function A returning instance // of a class or type info of a function which hasn't been created yet. // Thus check if member has already been created first. foreach (var cm in model.Classes) { - var cls = mf.ClassFactory.Construct(cm, null); + var cls = mf.ClassFactory.Construct(cm); _scopeVariables.DeclareVariable(cm.Name, cls, VariableSource.Declaration, mf.DefaultLocation); } foreach (var fm in model.Functions) { - var ft = mf.FunctionFactory.Construct(fm, null); + var ft = mf.FunctionFactory.Construct(fm); _scopeVariables.DeclareVariable(fm.Name, ft, VariableSource.Declaration, mf.DefaultLocation); } foreach (var vm in model.Variables) { - var v = mf.VariableFactory.Construct(vm, null); + var v = mf.VariableFactory.Construct(vm); _scopeVariables.DeclareVariable(vm.Name, v.Value, VariableSource.Declaration, mf.DefaultLocation); } // TODO: re-declare __doc__, __name__, etc. diff --git a/src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj b/src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj index e81ee3640..5f17847f4 100644 --- a/src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj +++ b/src/Caching/Impl/Microsoft.Python.Analysis.Caching.csproj @@ -21,6 +21,9 @@ + + + diff --git a/src/Caching/Impl/Models/FunctionModel.cs b/src/Caching/Impl/Models/FunctionModel.cs index 6049b5476..4608ff719 100644 --- a/src/Caching/Impl/Models/FunctionModel.cs +++ b/src/Caching/Impl/Models/FunctionModel.cs @@ -18,7 +18,6 @@ using System.Linq; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Utilities; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching.Models { diff --git a/src/Caching/Impl/Models/MemberModel.cs b/src/Caching/Impl/Models/MemberModel.cs index 1a4561e94..253ea6b54 100644 --- a/src/Caching/Impl/Models/MemberModel.cs +++ b/src/Caching/Impl/Models/MemberModel.cs @@ -13,8 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Core.Text; - namespace Microsoft.Python.Analysis.Caching.Models { internal abstract class MemberModel { public int Id { get; set; } diff --git a/src/Caching/Impl/Models/ModuleModel.cs b/src/Caching/Impl/Models/ModuleModel.cs index 883c774d8..7fe06f118 100644 --- a/src/Caching/Impl/Models/ModuleModel.cs +++ b/src/Caching/Impl/Models/ModuleModel.cs @@ -15,6 +15,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Caching.Factories; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -31,6 +32,7 @@ internal sealed class ModuleModel : MemberModel { public FunctionModel[] Functions { get; set; } public VariableModel[] Variables { get; set; } public ClassModel[] Classes { get; set; } + public TypeVarModel[] TypeVars { get; set; } /// /// Collection of new line information for conversion of linear spans @@ -43,18 +45,30 @@ internal sealed class ModuleModel : MemberModel { /// public int FileSize { get; set; } - // TODO: TypeVars, ... + public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options) { + var uniqueId = analysis.Document.GetUniqueId(services, options); + if(uniqueId == null) { + // Caching level setting does not permit this module to be persisted. + return null; + } - public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services) { var variables = new Dictionary(); var functions = new Dictionary(); var classes = new Dictionary(); + var typeVars = new Dictionary(); // Go directly through variables which names are listed in GetMemberNames // as well as variables that are declarations. var exportedNames = new HashSet(analysis.Document.GetMemberNames()); foreach (var v in analysis.GlobalScope.Variables - .Where(v => exportedNames.Contains(v.Name) || v.Source == VariableSource.Declaration || v.Source == VariableSource.Builtin)) { + .Where(v => exportedNames.Contains(v.Name) || + v.Source == VariableSource.Declaration || + v.Source == VariableSource.Builtin || + v.Source == VariableSource.Generic)) { + + if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name)) { + typeVars[v.Name] = TypeVarModel.FromGeneric(v); + } switch (v.Value) { case IPythonFunctionType ft when ft.IsLambda(): @@ -90,7 +104,6 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals } } - var uniqueId = analysis.Document.GetUniqueId(services); return new ModuleModel { Id = uniqueId.GetStableHash(), UniqueId = uniqueId, @@ -99,6 +112,7 @@ when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals Functions = functions.Values.ToArray(), Variables = variables.Values.ToArray(), Classes = classes.Values.ToArray(), + TypeVars = typeVars.Values.ToArray(), NewLines = analysis.Ast.NewLineLocations.Select(l => new NewLineModel { EndIndex = l.EndIndex, Kind = l.Kind diff --git a/src/Caching/Impl/Models/TypeVarModel.cs b/src/Caching/Impl/Models/TypeVarModel.cs index eeae11354..fba7dd8a1 100644 --- a/src/Caching/Impl/Models/TypeVarModel.cs +++ b/src/Caching/Impl/Models/TypeVarModel.cs @@ -14,11 +14,29 @@ // permissions and limitations under the License. using System.Diagnostics; +using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Caching.Models { - [DebuggerDisplay("t:{Name}")] - internal sealed class TypeVarModel { - public string Name { get; set; } + [DebuggerDisplay("TypeVar:{Name}")] + internal sealed class TypeVarModel: MemberModel { public string[] Constraints { get; set; } + public string Bound { get; set; } + public string Covariant { get; set; } + public string Contravariant { get; set; } + + public static TypeVarModel FromGeneric(IVariable v) { + var g = (IGenericTypeParameter)v.Value; + return new TypeVarModel { + Id = g.Name.GetStableHash(), + Name = g.Name, + Constraints = g.Constraints.Select(c => c.GetPersistentQualifiedName()).ToArray(), + Bound = g.Bound.GetPersistentQualifiedName(), + Covariant = g.Covariant.GetPersistentQualifiedName(), + Contravariant = g.Contravariant.GetPersistentQualifiedName() + }; + } } } diff --git a/src/Caching/Impl/ModuleDatabase.cs b/src/Caching/Impl/ModuleDatabase.cs index 3c7e2779e..97af51827 100644 --- a/src/Caching/Impl/ModuleDatabase.cs +++ b/src/Caching/Impl/ModuleDatabase.cs @@ -39,6 +39,7 @@ public ModuleDatabase(IServiceContainer services) { _services = services; _log = services.GetService(); _fs = services.GetService(); + var cfs = services.GetService(); _databaseFolder = Path.Combine(cfs.CacheFolder, $"analysis.v{_databaseFormatVersion}"); } @@ -54,6 +55,11 @@ public ModuleDatabase(IServiceContainer services) { /// Module storage state public ModuleStorageState TryCreateModule(string moduleName, string filePath, out IPythonModule module) { module = null; + + if (GetCachingLevel() == AnalysisCachingLevel.None) { + return ModuleStorageState.DoesNotExist; + } + // We don't cache results here. Module resolution service decides when to call in here // and it is responsible of overall management of the loaded Python modules. for (var retries = 50; retries > 0; --retries) { @@ -89,12 +95,16 @@ public ModuleStorageState TryCreateModule(string moduleName, string filePath, ou /// Writes module data to the database. /// public Task StoreModuleAnalysisAsync(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) - => Task.Run(() => StoreModuleAnalysis(analysis, cancellationToken)); + => Task.Run(() => StoreModuleAnalysis(analysis, cancellationToken), cancellationToken); /// /// Determines if module analysis exists in the storage. /// public bool ModuleExistsInStorage(string moduleName, string filePath) { + if(GetCachingLevel() == AnalysisCachingLevel.None) { + return false; + } + for (var retries = 50; retries > 0; --retries) { try { var dbPath = FindDatabaseFile(moduleName, filePath); @@ -107,7 +117,17 @@ public bool ModuleExistsInStorage(string moduleName, string filePath) { } private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken cancellationToken = default) { - var model = ModuleModel.FromAnalysis(analysis, _services); + var cachingLevel = GetCachingLevel(); + if(cachingLevel == AnalysisCachingLevel.None) { + return; + } + + var model = ModuleModel.FromAnalysis(analysis, _services, cachingLevel); + if (model == null) { + // Caching level setting does not permit this module to be persisted. + return; + } + Exception ex = null; for (var retries = 50; retries > 0; --retries) { cancellationToken.ThrowIfCancellationRequested(); @@ -146,7 +166,7 @@ private void StoreModuleAnalysis(IDocumentAnalysis analysis, CancellationToken c /// private string FindDatabaseFile(string moduleName, string filePath) { var interpreter = _services.GetService(); - var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services); + var uniqueId = ModuleUniqueId.GetUniqueId(moduleName, filePath, ModuleType.Specialized, _services, GetCachingLevel()); if (string.IsNullOrEmpty(uniqueId)) { return null; } @@ -170,5 +190,8 @@ private string FindDatabaseFile(string moduleName, string filePath) { dbPath = Path.Combine(_databaseFolder, $"{uniqueId}({pythonVersion.Major}).db"); return _fs.FileExists(dbPath) ? dbPath : null; } + + private AnalysisCachingLevel GetCachingLevel() + => _services.GetService()?.Options.AnalysisCachingLevel ?? AnalysisCachingLevel.None; } } diff --git a/src/Caching/Impl/ModuleUniqueId.cs b/src/Caching/Impl/ModuleUniqueId.cs index 879211599..9cf4fd376 100644 --- a/src/Caching/Impl/ModuleUniqueId.cs +++ b/src/Caching/Impl/ModuleUniqueId.cs @@ -25,19 +25,28 @@ namespace Microsoft.Python.Analysis.Caching { internal static class ModuleUniqueId { - public static string GetUniqueId(this IPythonModule module, IServiceContainer services) - => GetUniqueId(module.Name, module.FilePath, module.ModuleType, services); - - public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services) { - var interpreter = services.GetService(); - var fs = services.GetService(); + public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel) + => GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel); + public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services, AnalysisCachingLevel cachingLevel) { + if(cachingLevel == AnalysisCachingLevel.None) { + return null; + } if (moduleType == ModuleType.User) { // Only for tests. return $"{moduleName}"; } + var interpreter = services.GetService(); + var fs = services.GetService(); + var modulePathType = GetModulePathType(filePath, interpreter.ModuleResolution.LibraryPaths, fs); + switch(modulePathType) { + case PythonLibraryPathType.Site when cachingLevel < AnalysisCachingLevel.Library: + return null; + case PythonLibraryPathType.StdLib when cachingLevel < AnalysisCachingLevel.System: + return null; + } if (!string.IsNullOrEmpty(filePath) && modulePathType == PythonLibraryPathType.Site) { // Module can be a submodule of a versioned package. In this case we want to use diff --git a/src/Caching/Test/ClassesTests.cs b/src/Caching/Test/ClassesTests.cs index adaa23093..facbf28ff 100644 --- a/src/Caching/Test/ClassesTests.cs +++ b/src/Caching/Test/ClassesTests.cs @@ -52,15 +52,15 @@ def methodC(self): return False def methodB1(self): - return C() + return self.C() def methodB2(self): - return C().y + return self.C().y c = B().methodB1() "; var analysis = await GetAnalysisAsync(code); - var model = ModuleModel.FromAnalysis(analysis, Services); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); } diff --git a/src/Caching/Test/CoreTests.cs b/src/Caching/Test/CoreTests.cs index c5cd6ea1e..4b1adaafa 100644 --- a/src/Caching/Test/CoreTests.cs +++ b/src/Caching/Test/CoreTests.cs @@ -59,7 +59,7 @@ def func(): c = C() "; var analysis = await GetAnalysisAsync(code); - var model = ModuleModel.FromAnalysis(analysis, Services); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); } @@ -106,7 +106,7 @@ def func(a): ... .Which.Should().HaveSingleOverload() .Which.Should().HaveParameters(is3x ? new[] { "a", "b", "c" } : new[] { "a" }); - var model = ModuleModel.FromAnalysis(analysis, Services); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); Baseline.CompareToFile(GetBaselineFileNameWithSuffix(is3x ? "3" : "2"), json); } diff --git a/src/Caching/Test/Files/MemberLocations.json b/src/Caching/Test/Files/MemberLocations.json index 1269c8e2f..a5c60043f 100644 --- a/src/Caching/Test/Files/MemberLocations.json +++ b/src/Caching/Test/Files/MemberLocations.json @@ -241,6 +241,7 @@ } } ], + "TypeVars": [], "NewLines": [ { "EndIndex": 2, diff --git a/src/Caching/Test/Files/NestedClasses.json b/src/Caching/Test/Files/NestedClasses.json index 1ef88efe2..a74431141 100644 --- a/src/Caching/Test/Files/NestedClasses.json +++ b/src/Caching/Test/Files/NestedClasses.json @@ -80,7 +80,7 @@ "Id": 812, "Name": "c", "IndexSpan": { - "Start": 323, + "Start": 333, "Length": 1 } } @@ -181,7 +181,7 @@ "Id": 935009768, "Name": "methodB2", "IndexSpan": { - "Start": 282, + "Start": 287, "Length": 8 } } @@ -281,6 +281,7 @@ } } ], + "TypeVars": [], "NewLines": [ { "EndIndex": 2, @@ -351,31 +352,31 @@ "Kind": 3 }, { - "EndIndex": 272, + "EndIndex": 277, "Kind": 3 }, { - "EndIndex": 274, + "EndIndex": 279, "Kind": 3 }, { - "EndIndex": 299, + "EndIndex": 304, "Kind": 3 }, { - "EndIndex": 321, + "EndIndex": 331, "Kind": 3 }, { - "EndIndex": 323, + "EndIndex": 333, "Kind": 3 }, { - "EndIndex": 343, + "EndIndex": 353, "Kind": 3 } ], - "FileSize": 343, + "FileSize": 353, "Id": -2131035837, "Name": "module", "IndexSpan": null diff --git a/src/Caching/Test/Files/SmokeTest.json b/src/Caching/Test/Files/SmokeTest.json index be1c77d98..00b940866 100644 --- a/src/Caching/Test/Files/SmokeTest.json +++ b/src/Caching/Test/Files/SmokeTest.json @@ -199,6 +199,7 @@ } } ], + "TypeVars": [], "NewLines": [ { "EndIndex": 2, diff --git a/src/Caching/Test/Files/VersionHandling2.json b/src/Caching/Test/Files/VersionHandling2.json index 1e75eb30c..74643926a 100644 --- a/src/Caching/Test/Files/VersionHandling2.json +++ b/src/Caching/Test/Files/VersionHandling2.json @@ -94,6 +94,7 @@ } ], "Classes": [], + "TypeVars": [], "NewLines": [ { "EndIndex": 2, diff --git a/src/Caching/Test/Files/VersionHandling3.json b/src/Caching/Test/Files/VersionHandling3.json index 2ce8f4e38..707ef1020 100644 --- a/src/Caching/Test/Files/VersionHandling3.json +++ b/src/Caching/Test/Files/VersionHandling3.json @@ -106,6 +106,7 @@ } ], "Classes": [], + "TypeVars": [], "NewLines": [ { "EndIndex": 2, diff --git a/src/Caching/Test/LibraryModulesTests.cs b/src/Caching/Test/LibraryModulesTests.cs index cb319750c..fc1fe5d51 100644 --- a/src/Caching/Test/LibraryModulesTests.cs +++ b/src/Caching/Test/LibraryModulesTests.cs @@ -43,7 +43,7 @@ public void TestInitialize() public async Task Builtins() { var analysis = await GetAnalysisAsync(string.Empty); var builtins = analysis.Document.Interpreter.ModuleResolution.BuiltinsModule; - var model = ModuleModel.FromAnalysis(builtins.Analysis, Services); + var model = ModuleModel.FromAnalysis(builtins.Analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); @@ -52,25 +52,224 @@ public async Task Builtins() { dbModule.Should().HaveSameMembersAs(builtins); } + [TestMethod, Priority(0)] + public Task Ast() => TestModule("ast"); [TestMethod, Priority(0)] - public Task Sys() => TestModule("sys"); + public Task Asyncio() => TestModule("asyncio"); + + [TestMethod, Priority(0)] + public Task Base64() => TestModule("base64"); + + [TestMethod, Priority(0)] + public Task Bisect() => TestModule("bisect"); + + [TestMethod, Priority(0)] + public Task Calendar() => TestModule("calendar"); + + [TestMethod, Priority(0)] + public Task Collections() => TestModule("collections"); + + [TestMethod, Priority(0)] + public Task Concurrent() => TestModule("concurrent"); + + [TestMethod, Priority(0)] + public Task Crypt() => TestModule("crypt"); + + [TestMethod, Priority(0)] + [Ignore("_DRMapping type issue. Consider merge of module to stub so OrderedDict resolves to generic from the collections stub.")] + public Task Csv() => TestModule("csv"); + + [TestMethod, Priority(0)] + public Task CTypes() => TestModule("ctypes"); + + [TestMethod, Priority(0)] + public Task Curses() => TestModule("curses"); + + [TestMethod, Priority(0)] + public Task Dataclasses() => TestModule("dataclasses"); + + [TestMethod, Priority(0)] + public Task Datetime() => TestModule("datetime"); + + [TestMethod, Priority(0)] + public Task Dbm() => TestModule("dbm"); + + [TestMethod, Priority(0)] + public Task Distutils() => TestModule("distutils"); + + [TestMethod, Priority(0)] + public Task Email() => TestModule("email"); + + [TestMethod, Priority(0)] + public Task Encodings() => TestModule("encodings"); + + [TestMethod, Priority(0)] + public Task Enum() => TestModule("enum"); + + [TestMethod, Priority(0)] + public Task Filecmp() => TestModule("filecmp"); + + [TestMethod, Priority(0)] + public Task Fileinput() => TestModule("fileinput"); + + [TestMethod, Priority(0)] + public Task Fractions() => TestModule("fractions"); + + [TestMethod, Priority(0)] + public Task Ftplib() => TestModule("ftplib"); + + [TestMethod, Priority(0)] + public Task Functools() => TestModule("functools"); + + [TestMethod, Priority(0)] + public Task Genericpath() => TestModule("genericpath"); + + [TestMethod, Priority(0)] + public Task Glob() => TestModule("glob"); + + [TestMethod, Priority(0)] + public Task Gzip() => TestModule("gzip"); + + [TestMethod, Priority(0)] + public Task Hashlib() => TestModule("hashlib"); + + [TestMethod, Priority(0)] + public Task Heapq() => TestModule("heapq"); + + [TestMethod, Priority(0)] + public Task Html() => TestModule("html"); + + [TestMethod, Priority(0)] + public Task Http() => TestModule("http"); + + [TestMethod, Priority(0)] + public Task Importlib() => TestModule("importlib"); + + [TestMethod, Priority(0)] + public Task Imaplib() => TestModule("imaplib"); + + [TestMethod, Priority(0)] + public Task Imghdr() => TestModule("imghdr"); + + [TestMethod, Priority(0)] + public Task Inspect() => TestModule("inspect"); [TestMethod, Priority(0)] public Task Io() => TestModule("io"); [TestMethod, Priority(0)] - public Task Re() => TestModule("re"); + public Task Json() => TestModule("json"); + + [TestMethod, Priority(0)] + public Task Logging() => TestModule("logging"); + + [TestMethod, Priority(0)] + public Task Lzma() => TestModule("lzma"); + + [TestMethod, Priority(0)] + public Task Mailbox() => TestModule("mailbox"); + + [TestMethod, Priority(0)] + public Task Multiprocessing() => TestModule("multiprocessing"); [TestMethod, Priority(0)] public Task Os() => TestModule("os"); [TestMethod, Priority(0)] - public Task Logging() => TestModule("logging"); + public Task Pickle() => TestModule("pickle"); + + [TestMethod, Priority(0)] + public Task Pipes() => TestModule("pipes"); + + [TestMethod, Priority(0)] + public Task Pkgutil() => TestModule("pkgutil"); + + [TestMethod, Priority(0)] + [Ignore("Specialize Enum. See PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__)")] + public Task Plistlib() => TestModule("plistlib"); + + [TestMethod, Priority(0)] + public Task Pstats() => TestModule("pstats"); + + [TestMethod, Priority(0)] + public Task Pydoc() => TestModule("pydoc"); + + [TestMethod, Priority(0)] + public Task Queue() => TestModule("queue"); + + [TestMethod, Priority(0)] + public Task Random() => TestModule("random"); + + [TestMethod, Priority(0)] + public Task Re() => TestModule("re"); + + [TestMethod, Priority(0)] + public Task Reprlib() => TestModule("reprlib"); + + [TestMethod, Priority(0)] + public Task Signal() => TestModule("signal"); + + [TestMethod, Priority(0)] + public Task Site() => TestModule("site"); + + [TestMethod, Priority(0)] + public Task Socket() => TestModule("socket"); + + [TestMethod, Priority(0)] + public Task Sqlite3() => TestModule("sqlite3"); + + [TestMethod, Priority(0)] + public Task Statistics() => TestModule("statistics"); + + [TestMethod, Priority(0)] + public Task String() => TestModule("string"); + + [TestMethod, Priority(0)] + public Task Ssl() => TestModule("ssl"); + + [TestMethod, Priority(0)] + public Task Sys() => TestModule("sys"); + + [TestMethod, Priority(0)] + public Task Time() => TestModule("time"); + + [TestMethod, Priority(0)] + public Task Threading() => TestModule("threading"); + + [TestMethod, Priority(0)] + public Task Tkinter() => TestModule("tkinter"); + + [TestMethod, Priority(0)] + public Task Token() => TestModule("token"); + + [TestMethod, Priority(0)] + public Task Trace() => TestModule("trace"); [TestMethod, Priority(0)] public Task Types() => TestModule("types"); + [TestMethod, Priority(0)] + public Task Unittest() => TestModule("unittest"); + + [TestMethod, Priority(0)] + public Task Urllib() => TestModule("urllib"); + + [TestMethod, Priority(0)] + public Task Uuid() => TestModule("uuid"); + + [TestMethod, Priority(0)] + public Task Weakref() => TestModule("weakref"); + + [TestMethod, Priority(0)] + public Task Xml() => TestModule("xml"); + + [TestMethod, Priority(0)] + public Task XmlRpc() => TestModule("xmlrpc"); + + [TestMethod, Priority(0)] + public Task Zipfile() => TestModule("zipfile"); + [TestMethod, Priority(0)] public async Task Requests() { const string code = @" @@ -85,7 +284,7 @@ import requests } var rq = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("requests"); - var model = ModuleModel.FromAnalysis(rq.Analysis, Services); + var model = ModuleModel.FromAnalysis(rq.Analysis, Services, AnalysisCachingLevel.Library); var u = model.UniqueId; u.Should().Contain("(").And.EndWith(")"); @@ -99,7 +298,7 @@ import requests private async Task TestModule(string name) { var analysis = await GetAnalysisAsync($"import {name}"); var m = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(name); - var model = ModuleModel.FromAnalysis(m.Analysis, Services); + var model = ModuleModel.FromAnalysis(m.Analysis, Services, AnalysisCachingLevel.Library); CompareBaselineAndRestore(model, m); } diff --git a/src/Caching/Test/ReferencesTests.cs b/src/Caching/Test/ReferencesTests.cs index ca75fb8fe..867cd95c3 100644 --- a/src/Caching/Test/ReferencesTests.cs +++ b/src/Caching/Test/ReferencesTests.cs @@ -62,7 +62,7 @@ def methodB2(self): return 2 "; var analysis = await GetAnalysisAsync(code); - var model = ModuleModel.FromAnalysis(analysis, Services); + var model = ModuleModel.FromAnalysis(analysis, Services, AnalysisCachingLevel.Library); var json = ToJson(model); Baseline.CompareToFile(BaselineFileName, json); @@ -101,7 +101,7 @@ import logging "; var analysis = await GetAnalysisAsync(code); var logging = analysis.Document.Interpreter.ModuleResolution.GetImportedModule("logging"); - var model = ModuleModel.FromAnalysis(logging.Analysis, Services); + var model = ModuleModel.FromAnalysis(logging.Analysis, Services, AnalysisCachingLevel.Library); var dbModule = new PythonDbModule(model, logging.FilePath, Services); analysis.Document.Interpreter.ModuleResolution.SpecializeModule("logging", x => dbModule, replaceExisting: true); diff --git a/src/Core/Impl/Extensions/StringExtensions.cs b/src/Core/Impl/Extensions/StringExtensions.cs index a5044e33f..be0b1f899 100644 --- a/src/Core/Impl/Extensions/StringExtensions.cs +++ b/src/Core/Impl/Extensions/StringExtensions.cs @@ -303,6 +303,7 @@ public static string NormalizeLineEndings(this string s, string lineEnding = nul return string.Join(lineEnding, s.SplitLines()); } + [DebuggerStepThrough] public static int GetStableHash(this string s) { unchecked { var hash = 23; diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index 1ac34b14a..5e0de734f 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -80,6 +80,7 @@ private void HandleDiagnosticsChanges(JToken pythonSection, LanguageServerSettin var optionsProvider = _services.GetService(); optionsProvider.Options.KeepLibraryLocalVariables = GetSetting(memory, "keepLibraryLocalVariables", false); optionsProvider.Options.KeepLibraryAst = GetSetting(memory, "keepLibraryAst", false); + optionsProvider.Options.AnalysisCachingLevel = GetAnalysisCachingLevel(analysis); } internal static void HandleLintingOnOff(IServiceContainer services, bool linterEnabled) { @@ -100,5 +101,13 @@ internal static void HandleLintingOnOff(IServiceContainer services, bool linterE ds.Replace(m.Uri, entries, DiagnosticSource.Linter); } } + + private AnalysisCachingLevel GetAnalysisCachingLevel(JToken analysisKey) { + var s = GetSetting(analysisKey, "cachingLevel", "None"); + if (s.EqualsIgnoreCase("System")) { + return AnalysisCachingLevel.System; + } + return s.EqualsIgnoreCase("Library") ? AnalysisCachingLevel.Library : AnalysisCachingLevel.None; + } } } diff --git a/src/LanguageServer/Impl/Sources/DefinitionSource.cs b/src/LanguageServer/Impl/Sources/DefinitionSource.cs index 7a52c8b75..159277950 100644 --- a/src/LanguageServer/Impl/Sources/DefinitionSource.cs +++ b/src/LanguageServer/Impl/Sources/DefinitionSource.cs @@ -192,7 +192,7 @@ private Reference HandleImport(IDocumentAnalysis analysis, ImportStatement state private Reference TryFromVariable(string name, IDocumentAnalysis analysis, SourceLocation location, Node statement, out ILocatedMember definingMember) { definingMember = null; - var m = analysis.ExpressionEvaluator.LookupNameInScopes(name, out var scope); + var m = analysis.ExpressionEvaluator.LookupNameInScopes(name, out var scope, LookupOptions.All); if (m == null || scope.Module.ModuleType == ModuleType.Builtins || !(scope.Variables[name] is IVariable v)) { return null; }