Fix recursive expansion of null pointers in EE (#1550)

This PR addresses the issue of pointers to null being expandable.

Added a check to see if something is a null pointer if its a pointer
type and has the value of 0.

Added a test.

Addresses #698

Author: WardenGnaw

src/MIDebugEngine/AD7.Impl/AD7Property.cs

@@ -78,7 +78,7 @@ public DEBUG_PROPERTY_INFO ConstructDebugPropertyInfo(enum_DEBUGPROP_INFO_FLAGS
 
             if ((dwFields & enum_DEBUGPROP_INFO_FLAGS.DEBUGPROP_INFO_ATTRIB) != 0)
             {
-                if (variable.CountChildren != 0)
+                if (variable.CountChildren != 0 && !variable.IsNullPointer())
                 {
                     propertyInfo.dwAttrib |= enum_DBG_ATTRIB_FLAGS.DBG_ATTRIB_OBJ_IS_EXPANDABLE;
                 }
@@ -127,10 +127,10 @@ public DEBUG_PROPERTY_INFO ConstructDebugPropertyInfo(enum_DEBUGPROP_INFO_FLAGS
                 }
             }
 
-            // If the debugger has asked for the property, or the property has children (meaning it is a pointer in the sample)
+            // If the debugger has asked for the property, or the property has non-null children,
             // then set the pProperty field so the debugger can call back when the children are enumerated.
             if (((dwFields & enum_DEBUGPROP_INFO_FLAGS.DEBUGPROP_INFO_PROP) != 0) ||
-                (variable.CountChildren != 0))
+                (variable.CountChildren != 0 && !variable.IsNullPointer()))
             {
                 propertyInfo.pProperty = (IDebugProperty2)this;
                 propertyInfo.dwFields |= enum_DEBUGPROP_INFO_FLAGS.DEBUGPROP_INFO_PROP;

src/MIDebugEngine/Engine.Impl/Variables.cs

@@ -36,6 +36,7 @@ internal interface IVariableInformation : IDisposable
         string EvalDependentExpression(string expr);
         bool IsVisualized { get; }
         bool IsReadOnly();
+        bool IsNullPointer();
         enum_DEBUGPROP_INFO_FLAGS PropertyInfoFlags { get; set; }
         bool IsPreformatted { get; set; }
         string Address();
@@ -117,6 +118,30 @@ private static bool IsPointer(string typeName)
             return typeName.Trim().EndsWith("*", StringComparison.Ordinal);
         }
 
+        public bool IsNullPointer()
+        {
+            if (string.IsNullOrEmpty(TypeName) || !IsPointer(TypeName) || string.IsNullOrEmpty(Value))
+            {
+                return false;
+            }
+
+            string trimmed = Value.Trim();
+
+            // GDB may represent null as "0x0", "0x00", "0x0000000000000000", etc.
+            if (trimmed.StartsWith("0x", StringComparison.OrdinalIgnoreCase))
+            {
+                return long.TryParse(trimmed.Substring(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out long hexVal) && hexVal == 0;
+            }
+
+            // Decimal zero
+            if (long.TryParse(trimmed, NumberStyles.Integer, CultureInfo.InvariantCulture, out long decVal) && decVal == 0)
+            {
+                return true;
+            }
+
+            return false;
+        }
+
         public string FullName()   // Full expression used to re-compute the value
         {
             if (_fullname == null)

src/MIDebugEngine/Natvis.Impl/Natvis.cs

@@ -45,6 +45,7 @@ public SimpleWrapper(string name, AD7Engine engine, IVariableInformation underly
         public virtual bool IsVisualized { get { return Parent.IsVisualized; } }
         public virtual enum_DEBUGPROP_INFO_FLAGS PropertyInfoFlags { get; set; }
         public virtual bool IsReadOnly() => Parent.IsReadOnly();
+        public bool IsNullPointer() => Parent.IsNullPointer();
 
         public VariableInformation FindChildByName(string name) => Parent.FindChildByName(name);
         public string EvalDependentExpression(string expr) => Parent.EvalDependentExpression(expr);

test/CppTests/Tests/ExpressionTests.cs

@@ -516,6 +516,56 @@ public void SetExpressionOnVariable(ITestSettings settings)
 
         #endregion
 
+        [Theory]
+        [DependsOnTest(nameof(CompileKitchenSinkForExpressionTests))]
+        [RequiresTestSettings]
+        public void NullPointerNotExpandable(ITestSettings settings)
+        {
+            this.TestPurpose("Verify that a null pointer is not expandable in the variables view.");
+            this.WriteSettings(settings);
+
+            IDebuggee debuggee = SinkHelper.Open(this, settings.CompilerSettings, DebuggeeMonikers.KitchenSink.Expression);
+
+            using (IDebuggerRunner runner = CreateDebugAdapterRunner(settings))
+            {
+                this.Comment("Configure launch.");
+                runner.Launch(settings.DebuggerSettings, debuggee, "-fExpression");
+
+                this.Comment("Set a breakpoint after pStu is set to nullptr.");
+                runner.SetBreakpoints(debuggee.Breakpoints(SinkHelper.Expression, 53));
+
+                this.Comment("To start debugging and break.");
+                runner.Expects.StoppedEvent(StoppedReason.Breakpoint).AfterConfigurationDone();
+
+                using (IThreadInspector threadInspector = runner.GetThreadInspector())
+                {
+                    IFrameInspector currentFrame = threadInspector.Stack.First();
+
+                    this.Comment("Verify pStu is null.");
+                    currentFrame.AssertEvaluateAsNull("pStu", EvaluateContext.Watch);
+
+                    this.Comment("Verify null pointer is not expandable.");
+                    IVariableInspector pStuVar = currentFrame.GetVariable("pStu");
+                    Assert.True(
+                        pStuVar.VariablesReference == null || pStuVar.VariablesReference == 0,
+                        string.Format(CultureInfo.InvariantCulture,
+                            "Expected null pointer 'pStu' to not be expandable, but VariablesReference was {0}",
+                            pStuVar.VariablesReference));
+
+                    this.Comment("Verify non-null pointer student (on stack) is still expandable.");
+                    IVariableInspector studentVar = currentFrame.GetVariable("student");
+                    Assert.True(
+                        studentVar.VariablesReference != null && studentVar.VariablesReference > 0,
+                        "Expected non-null struct 'student' to be expandable.");
+                }
+
+                this.Comment("Run to completion.");
+                runner.Expects.TerminatedEvent().AfterContinue();
+
+                runner.DisconnectAndVerify();
+            }
+        }
+
         #region Private methods
 
         private static StackFrame[] GenerateFramesList(IDebuggerSettings debugger)