Fix microsoft#719: Scraper produces incorrect output for the Anaconda 2X builtin

Author: AlexanderSher

src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

@@ -86,6 +86,11 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV
                     return TryGetModulePossibleImport(possibleModuleImport, parent, location, out variableModule);
                 case ImplicitPackageImport packageImport:
                     return TryGetPackageFromImport(packageImport, parent, out variableModule);
+                case RelativeImportBeyondTopLevel importBeyondTopLevel:
+                    var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName);
+                    Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.Span, ErrorCodes.UnresolvedImport, Severity.Warning));
+                    variableModule = default;
+                    return false;
                 case ImportNotFound importNotFound:
                     var memberName = asNameExpression?.Name ?? importNotFound.FullName;
                     MakeUnresolvedImport(memberName, importNotFound.FullName, location);

src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs

@@ -41,7 +41,7 @@ public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable {
         private readonly AsyncAutoResetEvent _analysisRunningEvent = new AsyncAutoResetEvent();
         private readonly ProgressReporter _progress;
         private readonly ILogger _log;
-        private readonly int _maxTaskRunning = 8;
+        private readonly int _maxTaskRunning = Environment.ProcessorCount * 3;
         private int _runningTasks;
         private int _version;
 

src/Analysis/Ast/Impl/Resources.Designer.cs

@@ -312,6 +312,9 @@
   <data name="ErrorUnresolvedImport" xml:space="preserve">
     <value>unresolved import '{0}'</value>
   </data>
+  <data name="ErrorRelativeImportBeyondTopLevel" xml:space="preserve">
+    <value>Relative import '{0}' beyond top-level package</value>
+  </data>
   <data name="ErrorUseBeforeDef" xml:space="preserve">
     <value>'{0}' used before definition</value>
   </data>

src/Analysis/Ast/Impl/Resources.resx

@@ -28,9 +28,12 @@
 import tokenize
 import warnings
 
-try:
-    import builtins
-except ImportError:
+if sys.version_info >= (3, 0):
+    try:
+        import builtins
+    except ImportError:
+        import __builtin__ as builtins
+else:
     import __builtin__ as builtins
 
 try:

src/Analysis/Ast/Impl/scrape_module.py

@@ -19,6 +19,8 @@
 using Microsoft.Python.Analysis.Tests.FluentAssertions;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Parsing;
+using Microsoft.Python.Parsing.Tests;
 using Microsoft.Python.Tests.Utilities.FluentAssertions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using TestUtilities;
@@ -83,12 +85,19 @@ import sys
                 .And.HaveVariable("x").OfType(BuiltinTypeId.List);
         }
 
-        [TestMethod, Priority(0)]
-        public async Task BuiltinsTest() {
+        [DataRow(true, true)]
+        [DataRow(false, true)]
+        [DataRow(true, false)]
+        [DataRow(false, false)]
+        [DataTestMethod, Priority(0)]
+        public async Task BuiltinsTest(bool isPython3X, bool isAnaconda) {
             const string code = @"
 x = 1
 ";
-            var analysis = await GetAnalysisAsync(code);
+            var configuration = isPython3X 
+                ? isAnaconda ? PythonVersions.LatestAnaconda3X : PythonVersions.LatestAvailable3X 
+                : isAnaconda ? PythonVersions.LatestAnaconda2X : PythonVersions.LatestAvailable2X;
+            var analysis = await GetAnalysisAsync(code, configuration);
 
             var v = analysis.Should().HaveVariable("x").Which;
             var t = v.Value.GetPythonType();

src/Analysis/Ast/Test/BasicTests.cs

@@ -212,7 +212,7 @@ public async Task UnresolvedRelativeFromImportAs() {
             var d = analysis.Diagnostics.First();
             d.ErrorCode.Should().Be(ErrorCodes.UnresolvedImport);
             d.SourceSpan.Should().Be(1, 6, 1, 19);
-            d.Message.Should().Be(Resources.ErrorUnresolvedImport.FormatInvariant("nonexistent"));
+            d.Message.Should().Be(Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant("nonexistent"));
         }
     }
 }

src/Analysis/Ast/Test/ImportTests.cs

@@ -190,9 +190,9 @@ public IImportSearchResult GetImportsFromRelativePath(in string modulePath, in i
                 return default;
             }
 
-            if (parentCount > lastEdge.PathLength) {
+            if (parentCount >= lastEdge.PathLength) {
                 // Can't get outside of the root
-                return default;
+                return new RelativeImportBeyondTopLevel(string.Join(".", relativePath));
             }
 
             var fullNameList = relativePath.TakeWhile(n => !string.IsNullOrEmpty(n)).ToList();
@@ -208,7 +208,7 @@ public IImportSearchResult GetImportsFromRelativePath(in string modulePath, in i
 
                 return new ImportNotFound(new StringBuilder(lastEdge.Start.Name)
                     .Append(".")
-                    .Append(fullNameList)
+                    .Append(".", fullNameList)
                     .ToString());
             }
 

src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs

@@ -0,0 +1,23 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Python.Analysis.Core.DependencyResolution {
+    public class RelativeImportBeyondTopLevel : IImportSearchResult {
+        public string RelativeImportName { get; }
+        public RelativeImportBeyondTopLevel(string relativeImportName) {
+            RelativeImportName = relativeImportName;
+        }
+    }
+}

src/Analysis/Core/Impl/DependencyResolution/RelativeImportBeyondTopLevel.cs

@@ -903,6 +903,26 @@ public async Task FromDotInRoot() {
             result.Should().HaveNoCompletion();
         }
 
+        [TestMethod, Priority(0)]
+        public async Task FromDotInRootWithInitPy() {
+            var initPyPath = TestData.GetTestSpecificUri("__init__.py");
+            var module1Path = TestData.GetTestSpecificUri("module1.py");
+
+            var root = TestData.GetTestSpecificRootUri().AbsolutePath;
+            await CreateServicesAsync(root, PythonVersions.LatestAvailable3X);
+            var rdt = Services.GetService<IRunningDocumentTable>();
+
+            rdt.OpenDocument(initPyPath, string.Empty);
+            var module1 = rdt.OpenDocument(module1Path, "from .");
+            module1.Interpreter.ModuleResolution.GetOrLoadModule("__init__");
+
+            var analysis = await module1.GetAnalysisAsync(-1);
+
+            var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion);
+            var result = cs.GetCompletions(analysis, new SourceLocation(1, 7));
+            result.Should().HaveNoCompletion();
+        }
+
         [TestMethod, Priority(0)]
         public async Task FromDotInExplicitPackage() {
             var initPyPath = TestData.GetTestSpecificUri("package", "__init__.py");

src/LanguageServer/Test/CompletionTests.cs

@@ -81,6 +81,14 @@ public static class PythonVersions {
             Python27,
             Python27_x64).FirstOrDefault() ?? NotInstalled("v2");
 
+        public static InterpreterConfiguration LatestAnaconda3X => GetVersions(
+            Anaconda36,
+            Anaconda36_x64).FirstOrDefault() ?? NotInstalled("Anaconda v3");
+
+        public static InterpreterConfiguration LatestAnaconda2X => GetVersions(
+            Anaconda27,
+            Anaconda27_x64).FirstOrDefault() ?? NotInstalled("Anaconda v2");
+
         public static InterpreterConfiguration EarliestAvailable => EarliestAvailable2X ?? EarliestAvailable3X;
 
         public static InterpreterConfiguration EarliestAvailable3X => GetVersions(