Don't use alternate stack on illumos or Solaris.

When .NET translates SIGSEV to NullReferenceException, it does not return
from the signal handler. Instead it resumes execution at the catch handler
for the exception. This is not recommend by the manpage for sigaction(2):

> It is not recommended that [the ucontext] arg be used by the handler to
> restore the context from before the signal delivery.

The practical effect of resuming execution without returning from a handler
is that the alternate stack will not be used for subsequent signal delivery.
This is in contrast to the behavior on linux, which will always use the
alternate stack if the stack pointer at the time of fault does not fall on
the alternate stack.

Since the alternate stack is only usable for a single exception, don't
bother using it for any exceptions.

Author: AustinWise

src/coreclr/pal/src/exception/signal.cpp

@@ -190,13 +190,20 @@ BOOL SEHInitializeSignals(CorUnix::CPalThread *pthrCurrent, DWORD flags)
         handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv);
 #else
         handle_signal(SIGTRAP, sigtrap_handler, &g_previous_sigtrap);
+        int additionalFlagsForSigSegv = 0;
+#ifndef __sun
+        // On platforms that support it,
         // SIGSEGV handler runs on a separate stack so that we can handle stack overflow
-        handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv, SA_ONSTACK);
+        additionalFlagsForSigSegv |= SA_ONSTACK;
+#endif
+        handle_signal(SIGSEGV, sigsegv_handler, &g_previous_sigsegv, additionalFlagsForSigSegv);
 
+#ifndef __sun
         if (!pthrCurrent->EnsureSignalAlternateStack())
         {
             return FALSE;
         }
+#endif
 
         // Allocate the minimal stack necessary for handling stack overflow
         int stackOverflowStackSize = ALIGN_UP(sizeof(SignalHandlerWorkerReturnPoint), 16) + 7 * 4096;
@@ -336,7 +343,7 @@ Return :
 --*/
 bool IsRunningOnAlternateStack(void *context)
 {
-#if HAVE_MACH_EXCEPTIONS
+#if HAVE_MACH_EXCEPTIONS || defined(__sun)
     return false;
 #else
     bool isRunningOnAlternateStack;
@@ -644,6 +651,7 @@ static void sigsegv_handler(int code, siginfo_t *siginfo, void *context)
         // Now that we know the SIGSEGV didn't happen due to a stack overflow, execute the common
         // hardware signal handler on the original stack.
 
+#ifndef __sun
         if (GetCurrentPalThread() && IsRunningOnAlternateStack(context))
         {
             if (SwitchStackAndExecuteHandler(code, siginfo, context, 0 /* sp */)) // sp == 0 indicates execution on the original stack
@@ -652,6 +660,7 @@ static void sigsegv_handler(int code, siginfo_t *siginfo, void *context)
             }
         }
         else
+#endif
         {
             // The code flow gets here when the signal handler is not running on an alternate stack or when it wasn't created
             // by coreclr. In both cases, we execute the common_signal_handler directly.

src/coreclr/pal/src/include/pal/thread.hpp

@@ -288,8 +288,11 @@ namespace CorUnix
         void* m_stackBase;
         // Limit address of the stack of this thread
         void* m_stackLimit;
+
+#if !HAVE_MACH_EXCEPTIONS && !defined(__sun)
         // Signal handler's alternate stack to help with stack overflow
         void* m_alternateStack;
+#endif
 
         //
         // The thread entry routine (called from InternalCreateThread)
@@ -343,8 +346,10 @@ namespace CorUnix
             m_fStartStatus(FALSE),
             m_fStartStatusSet(FALSE),
             m_stackBase(NULL),
-            m_stackLimit(NULL),
-            m_alternateStack(NULL)
+            m_stackLimit(NULL)
+#if !HAVE_MACH_EXCEPTIONS && !defined(__sun)
+            ,m_alternateStack(NULL)
+#endif
         {
         };
 
@@ -588,7 +593,7 @@ namespace CorUnix
             m_pNext = pNext;
         };
 
-#if !HAVE_MACH_EXCEPTIONS
+#if !HAVE_MACH_EXCEPTIONS && !defined(__sun)
         BOOL
         EnsureSignalAlternateStack(
             void

src/coreclr/pal/src/init/sxs.cpp

@@ -64,7 +64,7 @@ AllocatePalThread(CPalThread **ppThread)
         goto exit;
     }
 
-#if !HAVE_MACH_EXCEPTIONS
+#if !HAVE_MACH_EXCEPTIONS && !defined(__sun)
     // Ensure alternate stack for SIGSEGV handling. Our SIGSEGV handler is set to
     // run on an alternate stack and the stack needs to be allocated per thread.
     if (!pThread->EnsureSignalAlternateStack())

src/coreclr/pal/src/thread/thread.cpp

@@ -157,7 +157,7 @@ static void InternalEndCurrentThreadWrapper(void *arg)
        will lock its own critical section */
     LOADCallDllMain(DLL_THREAD_DETACH, NULL);
 
-#if !HAVE_MACH_EXCEPTIONS
+#if !HAVE_MACH_EXCEPTIONS && !defined(__sun)
     pThread->FreeSignalAlternateStack();
 #endif // !HAVE_MACH_EXCEPTIONS
 
@@ -1667,7 +1667,7 @@ CPalThread::ThreadEntry(
     }
 #endif // HAVE_SCHED_GETAFFINITY && HAVE_SCHED_SETAFFINITY
 
-#if !HAVE_MACH_EXCEPTIONS
+#if !HAVE_MACH_EXCEPTIONS && !defined(__sun)
     if (!pThread->EnsureSignalAlternateStack())
     {
         ASSERT("Cannot allocate alternate stack for SIGSEGV!\n");
@@ -2390,7 +2390,7 @@ CPalThread::WaitForStartStatus(
     return m_fStartStatus;
 }
 
-#if !HAVE_MACH_EXCEPTIONS
+#if !HAVE_MACH_EXCEPTIONS && !defined(__sun)
 /*++
 Function :
     EnsureSignalAlternateStack