Undo loop optimization

src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs

@@ -15,6 +15,7 @@
 
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Linq;
 
 namespace Microsoft.Python.Analysis.Analyzer {
     internal static class ActivityTracker {

src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs

@@ -22,7 +22,7 @@
 
 namespace Microsoft.Python.Analysis.Analyzer {
     [DebuggerDisplay("{Name} : {FilePath}")]
-    public readonly struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
+    internal readonly struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
         public string Name { get; }
         public string FilePath { get; }
         public bool IsTypeshed { get; }
@@ -33,13 +33,14 @@ public AnalysisModuleKey(IPythonModule module) : this(
             module.IsTypeshed,
             IsNonUserAsDocumentModule(module)) { }
 
-        public AnalysisModuleKey(string name, string filePath, bool isTypeshed = false)
+        public AnalysisModuleKey(string name, string filePath, bool isTypeshed)
             : this(name, filePath, isTypeshed, false) { }
 
         private AnalysisModuleKey(string name, string filePath, bool isTypeshed, bool isNonUserAsDocument) {
             Name = name;
             FilePath = filePath;
             IsTypeshed = isTypeshed;
+            IsNonUserAsDocument = isNonUserAsDocument;
         }
 
         public AnalysisModuleKey GetNonUserAsDocumentKey() => new AnalysisModuleKey(Name, FilePath, IsTypeshed, true);
@@ -70,6 +71,8 @@ public void Deconstruct(out string moduleName, out string filePath, out bool isT
 
         public override string ToString() => $"{Name}({FilePath})";
 
+        public bool IsNonUserAsDocument { get; }
+
         private static bool IsNonUserAsDocumentModule(IPythonModule module)
             => (module.IsNonUserFile() || module.IsCompiled()) && module is IDocument document && document.IsOpen;
     }

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -40,9 +40,9 @@ internal abstract class AnalysisWalker : PythonWalker {
         public PythonAst Ast => Eval.Ast;
         protected ModuleSymbolTable SymbolTable => Eval.SymbolTable;
 
-        protected AnalysisWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) {
+        protected AnalysisWalker(ExpressionEval eval) {
             Eval = eval;
-            ImportHandler = new ImportHandler(this, importedVariableHandler);
+            ImportHandler = new ImportHandler(this);
             AssignmentHandler = new AssignmentHandler(this);
             LoopHandler = new LoopHandler(this);
             ConditionalHandler = new ConditionalHandler(this);

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -21,6 +21,11 @@ namespace Microsoft.Python.Analysis.Analyzer {
     /// Represents document that can be analyzed asynchronously.
     /// </summary>
     internal interface IAnalyzable {
+        /// <summary>
+        /// Returns object that can calculate dependencies of this entry.
+        /// </summary>
+        IDependencyProvider DependencyProvider { get; }
+
         /// <summary>
         /// Notifies document that analysis is about to begin.
         /// </summary>

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

@@ -90,13 +90,10 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
                 if (op.IsComparison()) {
                     return Interpreter.GetBuiltinType(BuiltinTypeId.Bool);
                 }
+
                 return UnknownType;
             }
 
-            var leftValue = left is IVariable v1 ? v1.Value : left;
-            var rightValue = right is IVariable v2 ? v2.Value : right;
-            var isInstance = leftValue is IPythonInstance || rightValue is IPythonInstance;
-
             var leftType = left.GetPythonType();
             var rightType = right.GetPythonType();
 
@@ -125,7 +122,7 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
 
             if (leftIsSupported && rightIsSupported) {
                 if (TryGetValueFromBuiltinBinaryOp(op, leftTypeId, rightTypeId, Interpreter.LanguageVersion.Is3x(), out var member)) {
-                    return isInstance ? new PythonInstance(member.GetPythonType()) : member;
+                    return member;
                 }
             }
 
@@ -140,10 +137,11 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
                     ret = CallOperator(op, left, leftType, right, rightType, expr, tryLeft: false);
                 }
 
-                if (ret.IsUnknown()) {
-                    ret = op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left;
+                if (!ret.IsUnknown()) {
+                    return ret;
                 }
-                return isInstance ? new PythonInstance(ret.GetPythonType()) : ret;
+
+                return op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left;
             }
 
             if (rightIsSupported) {

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

@@ -39,6 +39,9 @@ public T GetInScope<T>(string name, IScope scope) where T : class, IMember
         public void DeclareVariable(string name, IMember value, VariableSource source)
             => DeclareVariable(name, value, source, default(Location));
 
+        public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module)
+            => DeclareVariable(name, value, source, new Location(module));
+
         public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = true)
             => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite);
 
@@ -162,7 +165,7 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop
                     // node points to global scope, it is not a function or a class.
                     scope = gs;
                 } else {
-                    scope = outerScope.GetChildScope(node) as Scope;
+                    scope = outerScope.Children.OfType<Scope>().FirstOrDefault(s => s.Node == node);
                     if (scope == null) {
                         scope = new Scope(node, outerScope, Module);
                         outerScope.AddChildScope(scope);

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -235,8 +235,7 @@ private IMember GetValueFromName(NameExpression expr, LookupOptions options = Lo
         }
 
         private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOptions = LookupOptions.Normal) {
-            var memberName = expr?.Name;
-            if (expr?.Target == null || string.IsNullOrEmpty(memberName)) {
+            if (expr?.Target == null || string.IsNullOrEmpty(expr.Name)) {
                 return null;
             }
 
@@ -246,9 +245,8 @@ private IMember GetValueFromMember(MemberExpression expr, LookupOptions lookupOp
             }
 
             var type = m.GetPythonType();
-            var value = type?.GetMember(memberName);
-            var location = GetLocationOfName(expr);
-            type?.AddMemberReference(memberName, this, location);
+            var value = type?.GetMember(expr.Name);
+            type?.AddMemberReference(expr.Name, this, GetLocationOfName(expr));
 
             if (type is IPythonModule) {
                 return value;

src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs

@@ -14,7 +14,6 @@
 // permissions and limitations under the License.
 
 using System;
-using Microsoft.Python.Analysis.Analyzer.Handlers;
 using Microsoft.Python.Analysis.Analyzer.Symbols;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Parsing.Ast;
@@ -29,7 +28,7 @@ internal sealed class FunctionCallEvaluator: AnalysisWalker {
         private readonly FunctionDefinition _function;
         private IMember _result;
 
-        public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval, SimpleImportedVariableHandler.Instance) {
+        public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval) {
             _declaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule));
             _eval = eval ?? throw new ArgumentNullException(nameof(eval));
             _function = fd ?? throw new ArgumentNullException(nameof(fd));

src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

@@ -15,7 +15,6 @@
 
 using System.Diagnostics;
 using System.Linq;
-using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Parsing.Ast;
@@ -30,22 +29,14 @@ public void HandleAssignment(AssignmentStatement node, LookupOptions lookupOptio
             }
 
             // Filter out parenthesis expression in assignment because it makes no difference.
-            var lhs = node.Left.Select(s => s.RemoveParenthesis()).ToArray();
+            var lhs = node.Left.Select(s => s.RemoveParenthesis());
 
             // Note that this is handling assignments of the same value to multiple variables,
             // i.e. with "x = y = z = value", x/y/z are the items in lhs. If an expression looks
             // like "x, y, z = value", then "x, y, z" is a *single* lhs value and its unpacking
             // will be handled by AssignToExpr.
             var value = ExtractRhs(node.Right, lhs.FirstOrDefault(), lookupOptions);
             if (value != null) {
-                // Named tuple may get assigned to variable that has name different from the tuple itself.
-                // Then the name may end up conflicting with other types in module when stub merges with
-                // module types. For example, 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...)
-                // but there is also 'class TokenInfo(_TokenInfo)' so we have to use the variable name
-                // in order to avoid type naming conflicts.
-                if (value is ITypingNamedTupleType nt && lhs.Length == 1 && lhs[0] is NameExpression nex) {
-                    nt.SetName(nex.Name);
-                }
                 foreach (var expr in lhs) {
                     AssignToExpr(expr, value);
                 }

src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

@@ -59,21 +59,12 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor
 
             for (var i = 0; i < names.Count; i++) {
                 var memberName = names[i].Name;
-                if (string.IsNullOrEmpty(memberName)) {
-                    continue;
-                }
-
-                var nameExpression = asNames[i] ?? names[i];
-                var variableName = nameExpression?.Name ?? memberName;
-                if (!string.IsNullOrEmpty(variableName)) {
-                    DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression);
-                }
-
-                if (imports is IImportChildrenSource cs
-                    && cs.TryGetChildImport(memberName, out var csr)
-                    && HandleImportSearchResult(csr, variableModule, null, names[i], out var childModule)) {
-
-                    _importedVariableHandler.EnsureModule(childModule);
+                if (!string.IsNullOrEmpty(memberName)) {
+                    var nameExpression = asNames[i] ?? names[i];
+                    var variableName = nameExpression?.Name ?? memberName;
+                    if (!string.IsNullOrEmpty(variableName)) {
+                        DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression);
+                    }
                 }
             }
         }
@@ -83,11 +74,10 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, IImport
                 // from self import * won't define any new members
                 return;
             }
-
             // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore.
             var memberNames = imports is ImplicitPackageImport
                 ? variableModule.GetMemberNames()
-                : _importedVariableHandler.GetMemberNames(variableModule).ToArray();
+                : variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_"));
 
             foreach (var memberName in memberNames) {
                 DeclareVariable(variableModule, memberName, imports, memberName, importPosition, nameExpression);
@@ -114,36 +104,23 @@ private void DeclareVariable(PythonVariableModule variableModule, string memberN
                 value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
 
                 // First try exported or child submodules.
-                var member = variableModule.GetMember(memberName);
+                value = value ?? variableModule.GetMember(memberName);
 
                 // Value may be variable or submodule. If it is variable, we need it in order to add reference.
-                var variable = _importedVariableHandler.GetVariable(variableModule, memberName);
-
-                if (member is PythonVariableModule vm && vm.Equals(variable?.Value)) {
-                    // If member is submodule, use actual variable so it can be linked through for goto definition.
-                    value = variable;
-                } else if (value == null) {
-                    if (member is PythonVariableModule) {
-                        // If member is submodule, use it.
-                        value = member;
-                    } else if (variable?.Value != null) {
-                        // Otherwise use variable, if available so references can be linked.
-                        value = variable;
-                    } else if (member != null) {
-                        value = member;
-                    } else {
-                        value = Eval.UnknownType;
-                    }
-                }
-            }
+                var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
+                value = variable?.Value?.Equals(value) == true ? variable : value;
 
+                // If nothing is exported, variables are still accessible.
+                value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType;
+            }
+
             // Do not allow imported variables to override local declarations
             var canOverwrite = CanOverwriteVariable(variableName, importPosition, value);
-
+            
             // Do not declare references to '*'
             var locationExpression = nameLocation is NameExpression nex && nex.Name == "*" ? null : nameLocation;
             Eval.DeclareVariable(variableName, value, VariableSource.Import, locationExpression, canOverwrite);
-
+            
             // Make sure module is loaded and analyzed.
             if (value is IPythonModule m) {
                 ModuleResolution.GetOrLoadModule(m.Name);
@@ -155,8 +132,8 @@ private bool CanOverwriteVariable(string name, int importPosition, IMember newVa
             if (v == null) {
                 return true; // Variable does not exist
             }
-
-            if (newValue.IsUnknown()) {
+            
+            if(newValue.IsUnknown()) {
                 return false; // Do not overwrite potentially good value with unknowns.
             }
 

src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

@@ -29,12 +29,9 @@
 
 namespace Microsoft.Python.Analysis.Analyzer.Handlers {
     internal sealed partial class ImportHandler : StatementHandler {
-        private readonly IImportedVariableHandler _importedVariableHandler;
         private readonly Dictionary<string, PythonVariableModule> _variableModules = new Dictionary<string, PythonVariableModule>();
 
-        public ImportHandler(AnalysisWalker walker, in IImportedVariableHandler importedVariableHandler) : base(walker) {
-            _importedVariableHandler = importedVariableHandler;
-        }
+        public ImportHandler(AnalysisWalker walker) : base(walker) { }
 
         public bool HandleImport(ImportStatement node) {
             if (Module.ModuleType == ModuleType.Specialized) {
@@ -68,9 +65,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
                     lastModule = default;
                     break;
                 }
-
                 resolvedModules[i] = (nameExpression.Name, lastModule);
-                _importedVariableHandler.EnsureModule(lastModule);
             }
 
             // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz')
@@ -94,6 +89,15 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
                     Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]);
                 }
             }
+
+            // import a.b.c.d => declares a, b in the current module, c in b, d in c.
+            for (var i = 1; i < resolvedModules.Length - 1; i++) {
+                var (childName, childModule) = resolvedModules[i + 1];
+                if (!string.IsNullOrEmpty(childName) && childModule != null) {
+                    var parent = resolvedModules[i].module;
+                    parent?.AddChildModule(childName, childModule);
+                }
+            }
         }
 
         private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in Node location, out PythonVariableModule variableModule) {
@@ -108,7 +112,8 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV
                     return TryGetPackageFromImport(packageImport, parent, out variableModule);
                 case RelativeImportBeyondTopLevel importBeyondTopLevel:
                     var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName);
-                    Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
+                    Eval.ReportDiagnostics(Eval.Module.Uri,
+                        new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
                     variableModule = default;
                     return false;
                 case ImportNotFound importNotFound:
@@ -172,7 +177,7 @@ private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImpor
                         return false;
                 }
             }
-            
+
             return true;
         }
 
@@ -191,22 +196,24 @@ private void MakeUnresolvedImport(string variableName, string moduleName, Node l
         }
 
         private PythonVariableModule GetOrCreateVariableModule(in string fullName, in PythonVariableModule parentModule, in string memberName) {
-            if (!_variableModules.TryGetValue(fullName, out var variableModule)) {
-                variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
-                _variableModules[fullName] = variableModule;
+            if (_variableModules.TryGetValue(fullName, out var variableModule)) {
+                return variableModule;
             }
-
+
+            variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
+            _variableModules[fullName] = variableModule;
             parentModule?.AddChildModule(memberName, variableModule);
             return variableModule;
         }
 
         private PythonVariableModule GetOrCreateVariableModule(in IPythonModule module, in PythonVariableModule parentModule, in string memberName) {
             var moduleFullName = module.Name;
-            if (!_variableModules.TryGetValue(moduleFullName, out var variableModule)) {
-                variableModule = new PythonVariableModule(module);
-                _variableModules[moduleFullName] = variableModule;
+            if (_variableModules.TryGetValue(moduleFullName, out var variableModule)) {
+                return variableModule;
             }
 
+            variableModule = new PythonVariableModule(module);
+            _variableModules[moduleFullName] = variableModule;
             parentModule?.AddChildModule(memberName, variableModule);
             return variableModule;
         }