Improved tooltip and member doc display and other fixes

Python/Product/Analysis/Analysis.csproj

@@ -103,6 +103,7 @@
     <Compile Include="Interpreter\Ast\AstPythonProperty.cs" />
     <Compile Include="LanguageServer\CallbackEventArgs.cs" />
     <Compile Include="LanguageServer\DiagnosticsErrorSink.cs" />
+    <Compile Include="LanguageServer\DisplayTextBuilder.cs" />
     <Compile Include="LanguageServer\Enums.cs" />
     <Compile Include="LanguageServer\Messages.cs" />
     <Compile Include="LanguageServer\ParseQueue.cs" />

Python/Product/Analysis/Interpreter/Ast/AstPythonModule.cs

@@ -29,8 +29,9 @@ sealed class AstPythonModule : IPythonModule, IProjectEntry, ILocatedMember {
         private readonly IPythonInterpreter _interpreter;
         private readonly Dictionary<object, object> _properties;
         private readonly List<string> _childModules;
-        private bool _foundChildModules;
         private readonly Dictionary<string, IMember> _members;
+        private bool _foundChildModules;
+        private string _documentation = string.Empty;
 
         public static IPythonModule FromFile(
             IPythonInterpreter interpreter,
@@ -86,7 +87,6 @@ string moduleFullName
 
         internal AstPythonModule() {
             Name = string.Empty;
-            Documentation = string.Empty;
             FilePath = string.Empty;
             _properties = new Dictionary<object, object>();
             _childModules = new List<string>();
@@ -96,7 +96,7 @@ internal AstPythonModule() {
 
         internal AstPythonModule(string moduleName, IPythonInterpreter interpreter, PythonAst ast, string filePath) {
             Name = moduleName;
-            Documentation = ast.Documentation;
+            _documentation = ast.Documentation;
             FilePath = filePath;
             DocumentUri = ProjectEntry.MakeDocumentUri(FilePath);
             Locations = new[] { new LocationInfo(filePath, DocumentUri, 1, 1) };
@@ -126,7 +126,19 @@ internal void AddChildModule(string name, IPythonModule module) {
         }
 
         public string Name { get; }
-        public string Documentation { get; }
+        public string Documentation {
+            get {
+                if(_documentation == null) {
+                    _members.TryGetValue("__doc__", out var m);
+                    _documentation = (m as AstPythonStringLiteral)?.Value ?? string.Empty;
+                    if(string.IsNullOrEmpty(_documentation)) {
+                        _members.TryGetValue($"_{Name}", out m);
+                        _documentation = (m as AstNestedPythonModule)?.Documentation ?? string.Empty;
+                    }
+                }
+                return _documentation;
+            }
+        }
         public string FilePath { get; }
         public Uri DocumentUri { get; }
         public PythonMemberType MemberType => PythonMemberType.Module;

Python/Product/Analysis/Interpreter/Ast/AstScrapedPythonModule.cs

@@ -40,14 +40,21 @@ class AstScrapedPythonModule : IPythonModule
 
         public AstScrapedPythonModule(string name, string filePath) {
             Name = name ?? throw new ArgumentNullException(nameof(name));
-            _documentation = string.Empty;
             _filePath = filePath;
             _members = new Dictionary<string, IMember>();
         }
 
         public string Name { get; }
 
-        public string Documentation => _documentation;
+        public string Documentation {
+            get {
+                if (_documentation == null) {
+                    var m = GetMember(null, "__doc__") as AstPythonStringLiteral;
+                    _documentation = m != null ? m.Value : string.Empty;
+                }
+                return _documentation;
+            }
+        }
 
         public PythonMemberType MemberType => PythonMemberType.Module;
 
@@ -161,7 +168,7 @@ public void Imported(IModuleContext context) {
 
                     proc.Start();
                     var exitCode = proc.Wait(60000);
-                    
+
                     if (exitCode == null) {
                         proc.Kill();
                         fact.Log(TraceLevel.Error, "ScrapeTimeout", proc.FileName, proc.Arguments);

Python/Product/Analysis/LanguageServer/DisplayTextBuilder.cs

@@ -0,0 +1,134 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Microsoft.PythonTools.Analysis.Infrastructure;
+using Microsoft.PythonTools.Interpreter;
+
+namespace Microsoft.PythonTools.Analysis.LanguageServer {
+    sealed class DisplayTextBuilder {
+        private readonly RestTextConverter _textConverter = new RestTextConverter();
+
+        public string MakeHoverText(IEnumerable<AnalysisValue> values, string originalExpression, InformationDisplayOptions displayOptions) {
+            var result = new StringBuilder();
+            var documentations = new HashSet<string>();
+
+            foreach (var v in values) {
+                if(result.Length > 0) {
+                    result.AppendLine();
+                }
+
+                var doc = GetDocString(v);
+                doc = displayOptions.trimDocumentationLines ? LimitLines(doc) : doc;
+                if (string.IsNullOrEmpty(doc)) {
+                    continue;
+                }
+
+                if (documentations.Add(doc)) {
+                    result.AppendLine(doc);
+                }
+            }
+
+            var displayText = result.ToString();
+            var multiline = displayText.IndexOf('\n') >= 0;
+            if (displayOptions.trimDocumentationText && displayText.Length > displayOptions.maxDocumentationTextLength) {
+                displayText = displayText.Substring(0,
+                    Math.Max(3, displayOptions.maxDocumentationTextLength) - 3) + "...";
+
+                result.Clear();
+                result.Append(displayText);
+            }
+
+            if (!string.IsNullOrEmpty(originalExpression)) {
+                if (displayOptions.trimDocumentationText && originalExpression.Length > displayOptions.maxDocumentationTextLength) {
+                    originalExpression = originalExpression.Substring(0, 
+                        Math.Max(3, displayOptions.maxDocumentationTextLength) - 3) + "...";
+                }
+                if (multiline) {
+                    result.Insert(0, $"{originalExpression}:{Environment.NewLine}");
+                } else if (result.Length > 0) {
+                    result.Insert(0, $"{originalExpression}: ");
+                } else {
+                    result.Append($"{originalExpression}: <unknown type>");
+                }
+            }
+
+            return _textConverter.ToMarkdown(result.ToString());
+        }
+
+        public string MakeModuleHoverText(ModuleReference modRef) {
+            // Return module information
+            var contents = "{0} : module".FormatUI(modRef.Name);
+            if (!string.IsNullOrEmpty(modRef.Module?.Documentation)) {
+                contents += $"{Environment.NewLine}{Environment.NewLine}{modRef.Module.Documentation}";
+            }
+            return contents;
+        }
+
+        private static string GetDocString(AnalysisValue v) {
+            var doc = !string.IsNullOrEmpty(v.Documentation) ? v.Documentation : string.Empty;
+            var desc = !string.IsNullOrEmpty(v.Description) ? v.Description : string.Empty;
+            if (v.MemberType == PythonMemberType.Instance || v.MemberType == PythonMemberType.Constant) {
+                return !string.IsNullOrEmpty(desc) ? desc : doc;
+            }
+            return doc.Length > desc.Length ? doc : desc;
+        }
+
+        private static string LimitLines(
+            string str,
+            int maxLines = 30,
+            int charsPerLine = 200,
+            bool ellipsisAtEnd = true,
+            bool stopAtFirstBlankLine = false
+        ) {
+            if (string.IsNullOrEmpty(str)) {
+                return str;
+            }
+
+            var lineCount = 0;
+            var prettyPrinted = new StringBuilder();
+            var wasEmpty = true;
+
+            using (var reader = new StringReader(str)) {
+                for (var line = reader.ReadLine(); line != null && lineCount < maxLines; line = reader.ReadLine()) {
+                    if (string.IsNullOrWhiteSpace(line)) {
+                        if (wasEmpty) {
+                            continue;
+                        }
+                        wasEmpty = true;
+                        if (stopAtFirstBlankLine) {
+                            lineCount = maxLines;
+                            break;
+                        }
+                        lineCount += 1;
+                        prettyPrinted.AppendLine();
+                    } else {
+                        wasEmpty = false;
+                        lineCount += (line.Length / charsPerLine) + 1;
+                        prettyPrinted.AppendLine(line);
+                    }
+                }
+            }
+            if (ellipsisAtEnd && lineCount >= maxLines) {
+                prettyPrinted.AppendLine("...");
+            }
+            return prettyPrinted.ToString().Trim();
+        }
+    }
+}

Python/Product/Analysis/LanguageServer/RestTextConverter.cs

@@ -86,7 +86,7 @@ private string TransformLines(string docstring) {
 
             var sb = new StringBuilder();
             foreach(var s in _md) {
-                sb.AppendLine(s + " "); // Keep hard line breaks
+                sb.AppendLine(s + "  "); // Keep hard line breaks
             }
             return sb.ToString().Trim();
         }