Adding super as a specialized keyword (#1702)

* wip adding super as a specialized keyword

* moving super logic into its own type

* slimed down super

* -refactorings
-adding multi param support
-adding tests around gotoDefinition (still broken)

* refacting to better parse the second parameter of super

* adding more edge cases to tests, and fixes for them

* Updates for gotoDefinition  tests to succeed

* adding test for generic with super
refactoing python 2.7 two param super(..)  I was using the mro from the frist param always, but it should be the second.
- new helper functions IsInstanceOf() and IsSubClass()

* refactor
removing IsInstanceOf
removing nested if

* Now filter GetMemberNames using distinct
Added Distinct completion tests

* replacing Count() with Count

* adding copywrite and formatting

* refactor to condense null checks
minor fixups

* Adding tests for current missing super support

- multiple inheritance test
- persistence test

* updating persistence test
removing unused json for now

* changing ClassesWithSuper() to return super
so that it tests persistence of the super type

* Adding variable and member checks.

currently HaveSameMemberAs() is asserting in debug do to no model found for super.

* changing asser.inconclusive to [ignore]

* Persistence update:
- changing qualified name to :SuperType
- trimming the leading ":" from builttin types

* removing white space

* new test file SyperTypeTests
moving createSuper(..) to PythonSuperTypes

* removing duplicate tests

* adding namedtuple test

Author: bschnurr

src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs

@@ -17,6 +17,7 @@
 using Microsoft.Python.Analysis.Analyzer;
 using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
 
 namespace Microsoft.Python.Analysis {
@@ -77,5 +78,12 @@ public static bool GetSpecificType(this IPythonClassType cls, string paramName,
             }
             return specificType != null;
         }
+
+        public static bool IsSubClassOf(this IPythonClassType cls, IPythonClassType classToCheck) {
+            if (classToCheck == null) {
+                return false;
+            }
+            return cls?.Bases.MaybeEnumerate().Any(b => b.Equals(classToCheck)) ?? false;
+        }
     }
 }

src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs

@@ -174,9 +174,9 @@ private void SpecializeFunctions() {
             Analysis.SpecializeFunction("type", BuiltinsSpecializations.TypeInfo);
             Analysis.SpecializeFunction("vars", BuiltinsSpecializations.DictStringToObject);
 
+            Analysis.SpecializeFunction("super", BuiltinsSpecializations.Super);
             //SpecializeFunction(_builtinName, "range", RangeConstructor);
             //SpecializeFunction(_builtinName, "sorted", ReturnsListOfInputIterable);
-            //SpecializeFunction(_builtinName, "super", SpecialSuper);
         }
 
         private IReadOnlyList<ParameterInfo> OpenConstructor() {

src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs

@@ -13,6 +13,7 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
+using System;
 using System.Linq;
 using Microsoft.Python.Analysis.Specializations.Typing.Types;
 using Microsoft.Python.Analysis.Specializations.Typing.Values;
@@ -21,6 +22,7 @@
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Analysis.Values.Collections;
 using Microsoft.Python.Core.Text;
+using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Specializations {
     public static class BuiltinsSpecializations {
@@ -91,6 +93,40 @@ public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverlo
             return args.Count > 0 && args[0] is PythonCollection c ? c.Contents.FirstOrDefault() : null;
         }
 
+        public static IMember Super(IPythonModule declaringModule, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) {
+            var args = argSet.Values<IMember>();
+
+            if (args.Count == 0) {
+                //Zero argument form only works inside a class definition
+                foreach (var s in argSet.Eval.CurrentScope.EnumerateTowardsGlobal.Where(s => s.Node is ClassDefinition)) {
+                    var classType = s.Variables["__class__"].GetPythonType<IPythonClassType>();
+                    return PythonSuperType.CreateSuper(classType)?.CreateInstance(argSet); 
+                }
+                return null;
+            }
+
+            // If multiple arguments first argument is required
+            var firstCls = args.FirstOrDefault().GetPythonType<IPythonClassType>();
+            if (firstCls == null) {
+                return null;
+            }
+
+            // second argument optional
+            bool isUnbound = args.Count == 1;
+            if (isUnbound) {
+                return PythonSuperType.CreateSuper(firstCls)?.CreateInstance(argSet);
+            }
+
+            var secondCls = args[1].GetPythonType<IPythonClassType>();
+            if (secondCls?.Equals(firstCls) == true || 
+                secondCls?.IsSubClassOf(firstCls) == true) {
+                // We walk the mro of the second parameter looking for the first
+                return PythonSuperType.CreateSuper(secondCls, typeToFind: firstCls)?.CreateInstance(argSet);
+            }
+
+            return null;
+        }
+
         public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, IArgumentSet argSet, IndexSpan indexSpan) {
             var mode = argSet.GetArgumentValue<IPythonConstant>("mode");
 
@@ -141,7 +177,7 @@ public static IMember GetAttr(IPythonModule module, IPythonFunctionOverload over
             // getattr(a, 3.14)
             if (name == null) {
                 // TODO diagnostic error when second arg of getattr is not a string
-                return  module.Interpreter.UnknownType;
+                return module.Interpreter.UnknownType;
             }
 
             return o?.GetPythonType().GetMember(name) ?? def;

src/Analysis/Ast/Impl/Types/Definitions/IPythonSuperType.cs

@@ -0,0 +1,28 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+
+namespace Microsoft.Python.Analysis.Types {
+    /// <summary>
+    /// Represents Python class type definition.
+    /// </summary>
+    public interface IPythonSuperType {
+        /// <summary>
+        /// Python Method Resolution Order (MRO).
+        /// </summary>
+        IReadOnlyList<IPythonType> Mro { get; }
+    }
+}

src/Analysis/Ast/Impl/Types/PythonSuperType.cs

@@ -0,0 +1,76 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Core;
+
+namespace Microsoft.Python.Analysis.Types {
+    internal sealed class PythonSuperType : PythonType, IPythonSuperType {
+        /// <summary>
+        /// more info at https://docs.python.org/3/library/functions.html#super
+        /// </summary>
+        /// <param name="location"></param>
+        /// <param name="mro">Should be a list of IPythonType</param>
+        public PythonSuperType(IReadOnlyList<IPythonType> mro)
+            : base("super", new Location(), string.Empty, BuiltinTypeId.Type) {
+            Mro = mro;
+        }
+
+        public override string QualifiedName => $":SuperType[{string.Join(",", Mro.Select(t => t.QualifiedName))}]";
+
+        public IReadOnlyList<IPythonType> Mro { get; }
+
+        public override IMember GetMember(string name) => Mro.MaybeEnumerate().Select(c => c.GetMember(name)).ExcludeDefault().FirstOrDefault();
+
+        public override IEnumerable<string> GetMemberNames() => Mro.MaybeEnumerate().SelectMany(cls => cls.GetMemberNames()).Distinct();
+
+
+        /// <summary>
+        ///  This will return PythonSuperType with a mro list that starts with the next class in line in classType's mro, 
+        ///  or it will search classType's mro for typeToFild then build an mro list for the remaining classses after typeToFind.
+        /// </summary>
+        /// <param name="classType"></param>
+        /// <param name="typeToFind"></param>
+        /// <returns></returns>
+        internal static PythonSuperType CreateSuper(IPythonClassType classType, IPythonType typeToFind = null) {
+            var mro = classType?.Mro ?? Array.Empty<IPythonType>();
+            if (mro.Count == 0) {
+                return null;
+            }
+
+            // skip doing work if the newStartType is the first element in the callers mro
+            if (typeToFind?.Equals(classType.Mro.FirstOrDefault()) == false) {
+                var mroList = classType.Mro.ToList();
+                var start = mroList.FindIndex(0, t => t.Equals(typeToFind));
+                if (start >= 0) {
+                    mro = mroList.GetRange(start, mro.Count - start).ToArray();
+                } else {
+                    return null;  // typeToFind wasn't in the mro
+                }
+            }
+
+            var nextClassInLine = mro?.FirstOrDefault();
+            if (nextClassInLine != null) {
+                // Skip the first element, super's search starts at the next elemement in the mro for both super() and super(cls, typeToFind)
+                return new PythonSuperType(mro.Skip(1).ToArray());
+            }
+
+            return null;
+        }
+    }
+}
+