Merge persistence work into master

.gitattributes

@@ -0,0 +1 @@
+*.json -diff

build/Common.Build.Core.settings

@@ -9,8 +9,8 @@
   </PropertyGroup>
 
   <!-- This file is shared across desktop .NET and .NET Core builds. 
-        Please do not put Windows-specific or dekstop-specific settings
-        here (these incluse C#, C++ or VSSDK settings and targets -->
+        Please do not put Windows-specific or desktop-specific settings
+        here (these include C#, C++ or VS SDK settings and targets -->
   <PropertyGroup>
     <Configuration Condition="'$(Configuration)' == ''">Debug</Configuration>
     <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -13,11 +13,18 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using Microsoft.Python.Analysis.Dependencies;
+
 namespace Microsoft.Python.Analysis.Analyzer {
     /// <summary>
     /// Represents document that can be analyzed asynchronously.
     /// </summary>
     internal interface IAnalyzable {
+        /// <summary>
+        /// Returns object that can calculate dependencies of this entry.
+        /// </summary>
+        IDependencyProvider DependencyProvider { get; }
+
         /// <summary>
         /// Notifies document that analysis is about to begin.
         /// </summary>

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

@@ -17,7 +17,6 @@
 using System.Diagnostics;
 using System.Linq;
 using Microsoft.Python.Analysis.Documents;
-using Microsoft.Python.Analysis.Extensions;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs

@@ -62,7 +62,7 @@ public IMember GetValueFromList(ListExpression expression) {
                 var value = GetValueFromExpression(item) ?? UnknownType;
                 contents.Add(value);
             }
-            return PythonCollectionType.CreateList(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return PythonCollectionType.CreateList(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         public IMember GetValueFromDictionary(DictionaryExpression expression) {
@@ -72,7 +72,7 @@ public IMember GetValueFromDictionary(DictionaryExpression expression) {
                 var value = GetValueFromExpression(item.SliceStop) ?? UnknownType;
                 contents[key] = value;
             }
-            return new PythonDictionary(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return new PythonDictionary(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         private IMember GetValueFromTuple(TupleExpression expression) {
@@ -81,7 +81,7 @@ private IMember GetValueFromTuple(TupleExpression expression) {
                 var value = GetValueFromExpression(item) ?? UnknownType;
                 contents.Add(value);
             }
-            return PythonCollectionType.CreateTuple(Module.Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return PythonCollectionType.CreateTuple(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         public IMember GetValueFromSet(SetExpression expression) {
@@ -90,7 +90,7 @@ public IMember GetValueFromSet(SetExpression expression) {
                 var value = GetValueFromExpression(item) ?? UnknownType;
                 contents.Add(value);
             }
-            return PythonCollectionType.CreateSet(Interpreter, contents, exact: expression.Items.Count <= MaxCollectionSize);
+            return PythonCollectionType.CreateSet(Module, contents, exact: expression.Items.Count <= MaxCollectionSize);
         }
 
         public IMember GetValueFromGenerator(GeneratorExpression expression) {
@@ -110,14 +110,14 @@ public IMember GetValueFromComprehension(Comprehension node) {
                 switch (node) {
                     case ListComprehension lc:
                         var v1 = GetValueFromExpression(lc.Item) ?? UnknownType;
-                        return PythonCollectionType.CreateList(Interpreter, new[] { v1 });
+                        return PythonCollectionType.CreateList(Module, new[] { v1 });
                     case SetComprehension sc:
                         var v2 = GetValueFromExpression(sc.Item) ?? UnknownType;
-                        return PythonCollectionType.CreateSet(Interpreter, new[] { v2 });
+                        return PythonCollectionType.CreateSet(Module, new[] { v2 });
                     case DictionaryComprehension dc:
                         var k = GetValueFromExpression(dc.Key) ?? UnknownType;
                         var v = GetValueFromExpression(dc.Value) ?? UnknownType;
-                        return new PythonDictionary(new PythonDictionaryType(Interpreter), new Dictionary<IMember, IMember> { { k, v } });
+                        return new PythonDictionary(new PythonDictionaryType(Interpreter.ModuleResolution.BuiltinsModule), new Dictionary<IMember, IMember> { { k, v } });
                 }
 
                 return UnknownType;

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs

@@ -23,7 +23,7 @@
 
 namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
     internal sealed partial class ExpressionEval {
-        public IPythonInstance GetConstantFromLiteral(Expression expr, LookupOptions options) {
+        public IPythonInstance GetConstantFromLiteral(Expression expr) {
             if (expr is ConstantExpression ce) {
                 switch (ce.Value) {
                     case string s:

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs

@@ -62,11 +62,12 @@ private IMember GetValueFromGeneric(IMember target, Expression expr) {
         }
 
         /// <summary>
-        /// Returns whether the arguments to Generic are valid
+        /// Determines if arguments to Generic are valid
         /// </summary>
+        // TODO: move check to GenericClassBase. This requires extensive changes to SpecificTypeConstructor.
         private bool GenericClassParameterValid(IReadOnlyList<IGenericTypeParameter> genericTypeArgs, IReadOnlyList<IMember> args, Expression expr) {
-            // All arguments to Generic must be type parameters
-            // e.g. Generic[T, str] throws a runtime error
+            // All arguments to Generic must be type parameters	
+            // e.g. Generic[T, str] throws a runtime error	
             if (genericTypeArgs.Count != args.Count) {
                 ReportDiagnostics(Module.Uri, new DiagnosticsEntry(
                     Resources.GenericNotAllTypeParameters,
@@ -77,14 +78,14 @@ private bool GenericClassParameterValid(IReadOnlyList<IGenericTypeParameter> gen
                 return false;
             }
 
-            // All arguments to Generic must be distinct
+            // All arguments to Generic must be distinct	
             if (genericTypeArgs.Distinct().Count() != genericTypeArgs.Count) {
                 ReportDiagnostics(Module.Uri, new DiagnosticsEntry(
-                   Resources.GenericNotAllUnique,
-                   GetLocation(expr).Span,
-                   ErrorCodes.TypingGenericArguments,
-                   Severity.Warning,
-                   DiagnosticSource.Analysis));
+                    Resources.GenericNotAllUnique,
+                    GetLocation(expr).Span,
+                    ErrorCodes.TypingGenericArguments,
+                    Severity.Warning,
+                    DiagnosticSource.Analysis));
                 return false;
             }
 
@@ -97,24 +98,19 @@ private bool GenericClassParameterValid(IReadOnlyList<IGenericTypeParameter> gen
         /// (if the former) on specific type (if the latter).
         /// </summary>
         private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList<IMember> args, Expression expr) {
+            // TODO: move check to GenericClassBase. This requires extensive changes to SpecificTypeConstructor.
             if (gt.Name.EqualsOrdinal("Generic")) {
                 var genericTypeArgs = args.OfType<IGenericTypeParameter>().ToArray();
                 if (!GenericClassParameterValid(genericTypeArgs, args, expr)) {
                     return UnknownType;
                 }
-
-                // Generic[T1, T2, ...] expression. Create generic base for the class.
-                return new GenericClassParameter(genericTypeArgs, Module);
-            }
-
-            // For other types just use supplied arguments
-            if (args.Count > 0) {
-                return gt.CreateSpecificType(new ArgumentSet(args, expr, this));
+                // Generic[T1, T2, ...] expression. Create generic base for the class.	
+                return new GenericClassBase(genericTypeArgs, Module.Interpreter);
             }
 
-            return UnknownType;
+            // For other types just use supplied arguments	
+            return args.Count > 0 ? gt.CreateSpecificType(new ArgumentSet(args, expr, this)) : UnknownType;
         }
-
         private IReadOnlyList<IMember> EvaluateIndex(IndexExpression expr) {
             var indices = new List<IMember>();
             if (expr.Index is TupleExpression tex) {

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

@@ -105,9 +105,9 @@ private IMember GetValueFromBinaryOp(Expression expr) {
 
                 switch (leftTypeId) {
                     case BuiltinTypeId.List:
-                        return PythonCollectionType.CreateConcatenatedList(Module.Interpreter, lc, rc);
+                        return PythonCollectionType.CreateConcatenatedList(Module, lc, rc);
                     case BuiltinTypeId.Tuple:
-                        return PythonCollectionType.CreateConcatenatedTuple(Module.Interpreter, lc, rc);
+                        return PythonCollectionType.CreateConcatenatedTuple(Module, lc, rc);
                 }
             }
 

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -55,6 +55,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
         public ModuleSymbolTable SymbolTable { get; } = new ModuleSymbolTable();
         public IPythonType UnknownType => Interpreter.UnknownType;
         public Location DefaultLocation { get; }
+        public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule;
 
         public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty;
 
@@ -188,7 +189,7 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options = L
                     m = null;
                     break;
                 default:
-                    m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr, options);
+                    m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr);
                     break;
             }
             if (m == null) {
@@ -236,29 +237,32 @@ private IMember GetValueFromMember(MemberExpression expr) {
                 return null;
             }
 
-            IPythonInstance instance = null;
             var m = GetValueFromExpression(expr.Target);
-            if (m is IPythonType typeInfo) {
-                var member = typeInfo.GetMember(expr.Name);
-                // If container is class/type info rather than the instance, then the method is an unbound function.
-                // Example: C.f where f is a method of C. Compare to C().f where f is bound to the instance of C.
-                if (member is PythonFunctionType f && !f.IsStatic && !f.IsClassMethod) {
-                    f.AddReference(GetLocationOfName(expr));
-                    return f.ToUnbound();
-                }
-                instance = new PythonInstance(typeInfo);
+            if (m == null) {
+                return UnknownType;
             }
 
-            instance = instance ?? m as IPythonInstance;
-            var type = m?.GetPythonType(); // Try inner type
-
+            var type = m.GetPythonType();
             var value = type?.GetMember(expr.Name);
             type?.AddMemberReference(expr.Name, this, GetLocationOfName(expr));
 
             if (type is IPythonModule) {
                 return value;
             }
 
+            IPythonInstance instance = null;
+            if (m == type) {
+                // If container is class/type info rather than the instance, then the method is an unbound function.
+                // Example: C.f where f is a method of C. Compare to C().f where f is bound to the instance of C.
+                if (value is PythonFunctionType f && f.DeclaringType != null && !f.IsStatic && !f.IsClassMethod) {
+                    f.AddReference(GetLocationOfName(expr));
+                    return f.ToUnbound();
+                }
+                instance = new PythonInstance(type);
+            }
+
+            instance = instance ?? m as IPythonInstance;
+
             // Class type GetMember returns a type. However, class members are
             // mostly instances (consider self.x = 1, x is an instance of int).
             // However, it is indeed possible to have them as types, like in

src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

@@ -58,14 +58,14 @@ public void HandleAssignment(AssignmentStatement node) {
             }
 
             foreach (var ne in node.Left.OfType<NameExpression>()) {
+                IScope scope;
                 if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
-                    Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
+                    Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Nonlocal);
                     scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
                     continue;
                 }
-
                 if (Eval.CurrentScope.Globals[ne.Name] != null) {
-                    Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
+                    Eval.LookupNameInScopes(ne.Name, out scope, LookupOptions.Global);
                     scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
                     continue;
                 }

src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs

@@ -25,6 +25,6 @@ public ConditionalHandler(AnalysisWalker walker) : base(walker) {
         }
 
         public bool HandleIf(IfStatement node)
-            => node.WalkIfWithSystemConditions(Walker, Ast.LanguageVersion, _platformService.IsWindows);
+            => node.WalkIfWithSystemConditions(Walker, Ast.LanguageVersion, _platformService);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

@@ -68,11 +68,13 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor
                 if (!string.IsNullOrEmpty(memberName)) {
                     var nameExpression = asNames[i] ?? names[i];
                     var variableName = nameExpression?.Name ?? memberName;
-                    var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
-                    var exported = variable ?? variableModule.GetMember(memberName);
-                    var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
-                    // Do not allow imported variables to override local declarations
-                    Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex));
+                    if (!string.IsNullOrEmpty(variableName)) {
+                        var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
+                        var exported = variable ?? variableModule.GetMember(memberName);
+                        var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
+                        // Do not allow imported variables to override local declarations
+                        Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex));
+                    }
                 }
             }
         }
@@ -84,7 +86,7 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, bool is
             }
 
             // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore.
-            var memberNames = isImplicitPackage 
+            var memberNames = isImplicitPackage
                 ? variableModule.GetMemberNames()
                 : variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_"));
 
@@ -109,7 +111,7 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, bool is
 
         private bool CanOverwriteVariable(string name, int importPosition) {
             var v = Eval.CurrentScope.Variables[name];
-            if(v == null) {
+            if (v == null) {
                 return true; // Variable does not exist
             }
             // Allow overwrite if import is below the variable. Consider
@@ -118,10 +120,10 @@ private bool CanOverwriteVariable(string name, int importPosition) {
             //   from A import * # brings another x
             //   x = 3
             var references = v.References.Where(r => r.DocumentUri == Module.Uri).ToArray();
-            if(references.Length == 0) {
+            if (references.Length == 0) {
                 // No references to the variable in this file - the variable 
                 // is imported from another module. OK to overwrite.
-                return true; 
+                return true;
             }
             var firstAssignmentPosition = references.Min(r => r.Span.ToIndexSpan(Ast).Start);
             return firstAssignmentPosition < importPosition;
@@ -151,7 +153,7 @@ private void SpecializeFuture(FromImportStatement node) {
             var printNameExpression = node.Names.FirstOrDefault(n => n?.Name == "print_function");
             if (printNameExpression != null) {
                 var fn = new PythonFunctionType("print", new Location(Module), null, string.Empty);
-                var o = new PythonFunctionOverload(fn.Name, new Location(Module));
+                var o = new PythonFunctionOverload(fn, new Location(Module));
                 var parameters = new List<ParameterInfo> {
                     new ParameterInfo("*values", Interpreter.GetBuiltinType(BuiltinTypeId.Object), ParameterKind.List, null),
                     new ParameterInfo("sep", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.KeywordOnly, null),

src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

@@ -180,7 +180,7 @@ private PythonVariableModule GetOrCreateVariableModule(in string fullName, in Py
                 return variableModule;
             }
 
-            variableModule = new PythonVariableModule(fullName, Interpreter);
+            variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
             _variableModules[fullName] = variableModule;
             parentModule?.AddChildModule(memberName, variableModule);
             return variableModule;

src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs

@@ -15,12 +15,9 @@
 
 using System.Collections.Generic;
 using System.Linq;
-using Microsoft.Python.Analysis.Analyzer.Evaluation;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Documents;
-using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Values;
-using Microsoft.Python.Core;
 using Microsoft.Python.Core.Diagnostics;
 using Microsoft.Python.Parsing.Ast;