Improve ASP.NET CPU utilization by adjusting minWorkerThreads

aspnet/src/Application.cs

@@ -1,3 +1,6 @@
+using System;
+using System.Configuration;
+using System.Threading;
 using System.Web;
 using System.Web.Mvc;
 using System.Web.Routing;
@@ -25,6 +28,7 @@ private void Start()
         {
             Routes();
             Views();
+            Threads();
         }
 
         private void Routes()
@@ -57,6 +61,25 @@ private void Views()
             ViewEngines.Engines.Add(new RazorViewEngine { ViewLocationFormats = new[] { "~/Views/{0}.cshtml" } });
         }
 
+        private void Threads()
+        {
+            // To improve CPU utilization, increase the number of threads that the .NET thread pool expands by when
+            // a burst of requests come in. We could do this by editing machine.config/system.web/processModel/minWorkerThreads,
+            // but that seems too global a change, so we do it in code for just our AppPool. More info:
+            //
+            // http://support.microsoft.com/kb/821268
+            // http://blogs.msdn.com/b/tmarq/archive/2007/07/21/asp-net-thread-usage-on-iis-7-0-and-6-0.aspx
+            // http://blogs.msdn.com/b/perfworld/archive/2010/01/13/how-can-i-improve-the-performance-of-asp-net-by-adjusting-the-clr-thread-throttling-properties.aspx
+
+            int newMinWorkerThreads = Convert.ToInt32(ConfigurationManager.AppSettings["minWorkerThreadsPerLogicalProcessor"]);
+            if (newMinWorkerThreads > 0)
+            {
+                int minWorkerThreads, minCompletionPortThreads;
+                ThreadPool.GetMinThreads(out minWorkerThreads, out minCompletionPortThreads);
+                ThreadPool.SetMinThreads(Environment.ProcessorCount * newMinWorkerThreads, minCompletionPortThreads);
+            }
+        }
+
         public void Dispose()
         {
         }

aspnet/src/Web.config

@@ -25,6 +25,16 @@
       <provider invariantName="Npgsql" type="Npgsql.NpgsqlServices, Npgsql, Version=2.0.12.0"/>
     </providers>
   </entityFramework>
+  <appSettings>
+    <!-- Disable support for directly accessing *.cshtml/*.vbhtml files because that is a perf killer
+         and because we don't use such functionality. -->
+    <add key="webpages:Enabled" value="false" />
+    <!-- To fully saturate the CPUs, we need to allow the .NET thread pool to create many threads
+         when a large burst of requests come in. We do this by boosting the minWorkerThreads value
+         from the default of 1 per logical processor to 8 per logical processor. This seems to be
+         pretty conservative as http://support.microsoft.com/kb/821268 recommends 50.-->
+    <add key="minWorkerThreadsPerLogicalProcessor" value="8" />
+  </appSettings>
   <system.web>
     <!-- Show errors -->
     <customErrors mode="Off"/>

benchmarker.py

@@ -244,7 +244,7 @@ def __setup_server(self):
     try:
       if os.name == 'nt':
         return True
-      subprocess.check_call("sudo sysctl -w net.core.somaxconn=1024".rsplit(" "))
+      subprocess.check_call("sudo sysctl -w net.core.somaxconn=5000".rsplit(" "))
       subprocess.check_call("sudo -s ulimit -n 8192".rsplit(" "))
       subprocess.check_call("sudo sysctl net.ipv4.tcp_tw_reuse=1".rsplit(" "))
       subprocess.check_call("sudo sysctl net.ipv4.tcp_tw_recycle=1".rsplit(" "))
@@ -265,7 +265,7 @@ def __setup_server(self):
   def __setup_client(self):
     p = subprocess.Popen(self.ssh_string, stdin=subprocess.PIPE, shell=True)
     p.communicate("""
-      sudo sysctl -w net.core.somaxconn=1024
+      sudo sysctl -w net.core.somaxconn=5000
       sudo -s ulimit -n 8192
       sudo sysctl net.ipv4.tcp_tw_reuse=1
       sudo sysctl net.ipv4.tcp_tw_recycle=1

installer.py

@@ -1,5 +1,6 @@
 import subprocess
 import os
+import time
 
 class Installer:
 
@@ -31,7 +32,7 @@ def __install_server_software(self):
     self.__run_command("sudo apt-get install gcc-4.8 g++-4.8", True)
 
     self.__run_command("cp ../config/benchmark_profile ../../.bash_profile")
-    self.__run_command("sudo sh -c \"echo '*               soft    nofile          8192' >> /etc/security/limits.conf\"")
+    self.__run_command("sudo sh -c \"echo '*               -    nofile          8192' >> /etc/security/limits.conf\"")
 
     #######################################
     # Languages
@@ -110,7 +111,17 @@ def __install_server_software(self):
     # Perl
     #
 
-    self.__run_command("curl http://downloads.activestate.com/ActivePerl/releases/5.16.3.1603/ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746.tar.gz | tar xvz");
+    # Sometimes this HTTP server returns 404, so retry a few times until it works, but don't retry forever
+    tries = 0
+    while True:
+        self.__run_command("curl http://downloads.activestate.com/ActivePerl/releases/5.16.3.1603/ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746.tar.gz | tar xvz");
+        if os.path.exists(os.path.join('installs', 'ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746')):
+            break
+        tries += 1
+        if tries >= 30:
+            raise Exception('Could not download ActivePerl after many retries')
+        time.sleep(5)
+
     self.__run_command("sudo ./install.sh --license-accepted --prefix /opt/ActivePerl-5.16 --no-install-html", cwd="ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746", send_yes=True)
     self.__run_command("curl -L http://cpanmin.us | perl - --sudo App::cpanminus")
     self.__run_command("cpanm -f -S DBI DBD::mysql Kelp Dancer Mojolicious Kelp::Module::JSON::XS Dancer::Plugin::Database Starman Plack JSON Web::Simple DBD::Pg JSON::XS EV HTTP::Parser::XS Monoceros")
@@ -278,7 +289,7 @@ def __install_server_software(self):
     ##############################
     # Vert.x
     ##############################
-    self.__run_command("curl http://vertx.io/downloads/vert.x-1.3.1.final.tar.gz | tar xvz")
+    self.__run_command("curl http://vert-x.github.io/vertx-downloads/downloads/vert.x-1.3.1.final.tar.gz | tar xvz")
 
     ##############################
     # Yesod
@@ -322,7 +333,7 @@ def __install_client_software(self):
     ##############################
     yes | sudo apt-get update
     yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev postgresql
-    sudo sh -c "echo '*               soft    nofile          8192' >> /etc/security/limits.conf"
+    sudo sh -c "echo '*               -    nofile          8192' >> /etc/security/limits.conf"
 
     sudo mkdir -p /ssd
     sudo mkdir -p /ssd/log