Improve handling of circular dependencies

src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs

@@ -15,7 +15,6 @@
 
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 
 namespace Microsoft.Python.Analysis.Analyzer {
     internal static class ActivityTracker {

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -39,9 +39,9 @@ internal abstract class AnalysisWalker : PythonWalker {
         public PythonAst Ast => Eval.Ast;
         protected ModuleSymbolTable SymbolTable => Eval.SymbolTable;
 
-        protected AnalysisWalker(ExpressionEval eval) {
+        protected AnalysisWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) {
             Eval = eval;
-            ImportHandler = new ImportHandler(this);
+            ImportHandler = new ImportHandler(this, importedVariableHandler);
             AssignmentHandler = new AssignmentHandler(this);
             LoopHandler = new LoopHandler(this);
             ConditionalHandler = new ConditionalHandler(this);

src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

@@ -71,5 +71,12 @@ public interface IPythonAnalyzer {
         /// Fires when analysis is complete.
         /// </summary>
         event EventHandler<AnalysisCompleteEventArgs> AnalysisComplete;
+
+        /// <summary>
+        /// Attempts to restore modules analysis from database.
+        /// </summary>
+        /// <param name="module"></param>
+        /// <returns></returns>
+        IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

@@ -40,9 +40,6 @@ public T GetInScope<T>(string name, IScope scope) where T : class, IMember
         public void DeclareVariable(string name, IMember value, VariableSource source)
             => DeclareVariable(name, value, source, default(Location));
 
-        public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module)
-            => DeclareVariable(name, value, source, new Location(module));
-
         public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = true)
             => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite);
 

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -60,7 +60,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
         public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty;
 
         public Location GetLocationOfName(Node node) {
-            if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) {
+            if (node == null || Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library) {
                 return DefaultLocation;
             }
 
@@ -233,7 +233,8 @@ private IMember GetValueFromName(NameExpression expr, LookupOptions options = Lo
         }
 
         private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOptions = LookupOptions.Normal) {
-            if (expr?.Target == null || string.IsNullOrEmpty(expr.Name)) {
+            var memberName = expr?.Name;
+            if (expr?.Target == null || string.IsNullOrEmpty(memberName)) {
                 return null;
             }
 
@@ -243,8 +244,9 @@ private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOp
             }
 
             var type = m.GetPythonType();
-            var value = type?.GetMember(expr.Name);
-            type?.AddMemberReference(expr.Name, this, GetLocationOfName(expr));
+            var value = type?.GetMember(memberName);
+            var location = GetLocationOfName(expr);
+            type?.AddMemberReference(memberName, this, location);
 
             if (type is IPythonModule) {
                 return value;

src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs

@@ -14,6 +14,7 @@
 // permissions and limitations under the License.
 
 using System;
+using Microsoft.Python.Analysis.Analyzer.Handlers;
 using Microsoft.Python.Analysis.Analyzer.Symbols;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Parsing.Ast;
@@ -28,7 +29,7 @@ internal sealed class FunctionCallEvaluator: AnalysisWalker {
         private readonly FunctionDefinition _function;
         private IMember _result;
 
-        public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval) {
+        public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval, SimpleImportedVariableHandler.Instance) {
             _declaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule));
             _eval = eval ?? throw new ArgumentNullException(nameof(eval));
             _function = fd ?? throw new ArgumentNullException(nameof(fd));

src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

@@ -59,12 +59,21 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor
 
             for (var i = 0; i < names.Count; i++) {
                 var memberName = names[i].Name;
-                if (!string.IsNullOrEmpty(memberName)) {
-                    var nameExpression = asNames[i] ?? names[i];
-                    var variableName = nameExpression?.Name ?? memberName;
-                    if (!string.IsNullOrEmpty(variableName)) {
-                        DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression);
-                    }
+                if (string.IsNullOrEmpty(memberName)) {
+                    continue;
+                }
+
+                var nameExpression = asNames[i] ?? names[i];
+                var variableName = nameExpression?.Name ?? memberName;
+                if (!string.IsNullOrEmpty(variableName)) {
+                    DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression);
+                }
+
+                if (imports is IImportChildrenSource cs
+                    && cs.TryGetChildImport(memberName, out var csr)
+                    && HandleImportSearchResult(csr, variableModule, null, names[i], out var childModule)) {
+
+                    _importedVariableHandler.EnsureModule(childModule);
                 }
             }
         }
@@ -74,10 +83,11 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, IImport
                 // from self import * won't define any new members
                 return;
             }
+
             // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore.
             var memberNames = imports is ImplicitPackageImport
                 ? variableModule.GetMemberNames()
-                : variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_"));
+                : _importedVariableHandler.GetMemberNames(variableModule).ToArray();
 
             foreach (var memberName in memberNames) {
                 DeclareVariable(variableModule, memberName, imports, memberName, importPosition, nameExpression);
@@ -104,23 +114,36 @@ private void DeclareVariable(PythonVariableModule variableModule, string memberN
                 value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
 
                 // First try exported or child submodules.
-                value = value ?? variableModule.GetMember(memberName);
+                var member = variableModule.GetMember(memberName);
 
                 // Value may be variable or submodule. If it is variable, we need it in order to add reference.
-                var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
-                value = variable?.Value?.Equals(value) == true ? variable : value;
-
-                // If nothing is exported, variables are still accessible.
-                value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType;
+                var variable = _importedVariableHandler.GetVariable(variableModule, memberName);
+
+                if (member is PythonVariableModule vm && vm.Equals(variable?.Value)) {
+                    // If member is submodule, use actual variable so it can be linked through for goto definition.
+                    value = variable;
+                } else if (value == null) {
+                    if (member is PythonVariableModule) {
+                        // If member is submodule, use it.
+                        value = member;
+                    } else if (variable?.Value != null) {
+                        // Otherwise use variable, if available so references can be linked.
+                        value = variable;
+                    } else if (member != null) {
+                        value = member;
+                    } else {
+                        value = Eval.UnknownType;
+                    }
+                }
             }
-            
+
             // Do not allow imported variables to override local declarations
             var canOverwrite = CanOverwriteVariable(variableName, importPosition, value);
-            
+
             // Do not declare references to '*'
             var locationExpression = nameLocation is NameExpression nex && nex.Name == "*" ? null : nameLocation;
             Eval.DeclareVariable(variableName, value, VariableSource.Import, locationExpression, canOverwrite);
-            
+
             // Make sure module is loaded and analyzed.
             if (value is IPythonModule m) {
                 ModuleResolution.GetOrLoadModule(m.Name);
@@ -132,8 +155,8 @@ private bool CanOverwriteVariable(string name, int importPosition, IMember newVa
             if (v == null) {
                 return true; // Variable does not exist
             }
-            
-            if(newValue.IsUnknown()) {
+
+            if (newValue.IsUnknown()) {
                 return false; // Do not overwrite potentially good value with unknowns.
             }
 

src/Analysis/Ast/Impl/Analyzer/Handlers/IImportedVariableHandler.cs

@@ -0,0 +1,26 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using Microsoft.Python.Analysis.Modules;
+using Microsoft.Python.Analysis.Values;
+
+namespace Microsoft.Python.Analysis.Analyzer.Handlers {
+    internal interface IImportedVariableHandler {
+        IEnumerable<string> GetMemberNames(PythonVariableModule variableModule);
+        IVariable GetVariable(in PythonVariableModule module, in string name);
+        void EnsureModule(in PythonVariableModule module);
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

@@ -29,9 +29,12 @@
 
 namespace Microsoft.Python.Analysis.Analyzer.Handlers {
     internal sealed partial class ImportHandler : StatementHandler {
+        private readonly IImportedVariableHandler _importedVariableHandler;
         private readonly Dictionary<string, PythonVariableModule> _variableModules = new Dictionary<string, PythonVariableModule>();
 
-        public ImportHandler(AnalysisWalker walker) : base(walker) { }
+        public ImportHandler(AnalysisWalker walker, in IImportedVariableHandler importedVariableHandler) : base(walker) {
+            _importedVariableHandler = importedVariableHandler;
+        }
 
         public bool HandleImport(ImportStatement node) {
             if (Module.ModuleType == ModuleType.Specialized) {
@@ -65,7 +68,9 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
                     lastModule = default;
                     break;
                 }
+
                 resolvedModules[i] = (nameExpression.Name, lastModule);
+                _importedVariableHandler.EnsureModule(lastModule);
             }
 
             // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz')
@@ -89,15 +94,6 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
                     Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]);
                 }
             }
-
-            // import a.b.c.d => declares a, b in the current module, c in b, d in c.
-            for (var i = 1; i < resolvedModules.Length - 1; i++) {
-                var (childName, childModule) = resolvedModules[i + 1];
-                if (!string.IsNullOrEmpty(childName) && childModule != null) {
-                    var parent = resolvedModules[i].module;
-                    parent?.AddChildModule(childName, childModule);
-                }
-            }
         }
 
         private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in Node location, out PythonVariableModule variableModule) {
@@ -112,8 +108,7 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV
                     return TryGetPackageFromImport(packageImport, parent, out variableModule);
                 case RelativeImportBeyondTopLevel importBeyondTopLevel:
                     var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName);
-                    Eval.ReportDiagnostics(Eval.Module.Uri,
-                        new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
+                    Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
                     variableModule = default;
                     return false;
                 case ImportNotFound importNotFound:
@@ -177,7 +172,7 @@ private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImpor
                         return false;
                 }
             }
-
+            
             return true;
         }
 
@@ -196,24 +191,22 @@ private void MakeUnresolvedImport(string variableName, string moduleName, Node l
         }
 
         private PythonVariableModule GetOrCreateVariableModule(in string fullName, in PythonVariableModule parentModule, in string memberName) {
-            if (_variableModules.TryGetValue(fullName, out var variableModule)) {
-                return variableModule;
+            if (!_variableModules.TryGetValue(fullName, out var variableModule)) {
+                variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
+                _variableModules[fullName] = variableModule;
             }
-
-            variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
-            _variableModules[fullName] = variableModule;
+
             parentModule?.AddChildModule(memberName, variableModule);
             return variableModule;
         }
 
         private PythonVariableModule GetOrCreateVariableModule(in IPythonModule module, in PythonVariableModule parentModule, in string memberName) {
             var moduleFullName = module.Name;
-            if (_variableModules.TryGetValue(moduleFullName, out var variableModule)) {
-                return variableModule;
+            if (!_variableModules.TryGetValue(moduleFullName, out var variableModule)) {
+                variableModule = new PythonVariableModule(module);
+                _variableModules[moduleFullName] = variableModule;
             }
 
-            variableModule = new PythonVariableModule(module);
-            _variableModules[moduleFullName] = variableModule;
             parentModule?.AddChildModule(memberName, variableModule);
             return variableModule;
         }