From 69cd3564bf8bd8c19019da59ea706b1b6bb91881 Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Thu, 23 May 2013 20:28:25 -0700 Subject: [PATCH 01/11] Set hard open file limit in addition to soft limit On Ubuntu Server, the hard limit is 4096, which is less than the soft limit of 8192 that we're trying to use. Thus, to really use a soft limit of 8192, we need to also boost the hard limit to 8192. --- installer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer.py b/installer.py index 45a9e2fb26d..dbd54d90da9 100644 --- a/installer.py +++ b/installer.py @@ -31,7 +31,7 @@ def __install_server_software(self): self.__run_command("sudo apt-get install gcc-4.8 g++-4.8", True) self.__run_command("cp ../config/benchmark_profile ../../.bash_profile") - self.__run_command("sudo sh -c \"echo '* soft nofile 8192' >> /etc/security/limits.conf\"") + self.__run_command("sudo sh -c \"echo '* - nofile 8192' >> /etc/security/limits.conf\"") ####################################### # Languages @@ -365,7 +365,7 @@ def __install_client_software(self): ############################## yes | sudo apt-get update yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev postgresql - sudo sh -c "echo '* soft nofile 8192' >> /etc/security/limits.conf" + sudo sh -c "echo '* - nofile 8192' >> /etc/security/limits.conf" sudo mkdir -p /ssd sudo mkdir -p /ssd/log From 814affdc118764c01822f961466a6bcdaf9c8e99 Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Thu, 23 May 2013 22:50:49 -0700 Subject: [PATCH 02/11] Configure apache2 not to auto-start at boot Tests expect to be able to start their own desired http server on Apache's configured port. --- installer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/installer.py b/installer.py index dbd54d90da9..6c6cd84c99b 100644 --- a/installer.py +++ b/installer.py @@ -184,6 +184,7 @@ def __install_server_software(self): self.__run_command("sudo mv /etc/apache2/ports.conf /etc/apache2/ports.conf.orig") self.__run_command("sudo sh -c \"cat ../config/ports.conf > /etc/apache2/ports.conf\"") self.__run_command("sudo /etc/init.d/apache2 stop") + self.__run_command("sudo update-rc.d apache2 disable") # # Nginx From 1d989d46e00bf7b540b133aaa85071b97690a084 Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Sat, 25 May 2013 19:53:58 -0700 Subject: [PATCH 03/11] Fix MySQL connection problems by making back_log=5000 work by setting net.core.somaxconn=5000 When running tests that accessed MySQL, I was getting errors like: BENCHMARKING Query ... [MySQL] 2013/05/23 00:57:04 packets.go:31: EOF 2013/05/23 00:57:04 Error scanning world row: driver: bad connection exit status 1 Debugging this showed that the web server box had more connections to MySQL than expected (i.e. netstat on the MySQL box showed 100 TCP connections, but the web server box showed 1200). So there was some sort of TCP problem, perhaps lost ACKs as described in http://www.evanjones.ca/tcp-stuck-connection-mystery.html . Looking at the MySQL documentation for back_log (http://dev.mysql.com/doc/refman/5.0/en/server-system-variables.html#sysvar_back_log), it suggests that the OS setting needs to be greater than or equal to back_log. Thus, we need to set net.core.somaxconn as big as our back_log value. I tried this and the connection problems went away. --- benchmarker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarker.py b/benchmarker.py index f94aa8d10d7..bd5d88253c3 100644 --- a/benchmarker.py +++ b/benchmarker.py @@ -244,7 +244,7 @@ def __setup_server(self): try: if os.name == 'nt': return True - subprocess.check_call("sudo sysctl -w net.core.somaxconn=1024".rsplit(" ")) + subprocess.check_call("sudo sysctl -w net.core.somaxconn=5000".rsplit(" ")) subprocess.check_call("sudo -s ulimit -n 8192".rsplit(" ")) subprocess.check_call("sudo sysctl net.ipv4.tcp_tw_reuse=1".rsplit(" ")) subprocess.check_call("sudo sysctl net.ipv4.tcp_tw_recycle=1".rsplit(" ")) @@ -265,7 +265,7 @@ def __setup_server(self): def __setup_client(self): p = subprocess.Popen(self.ssh_string, stdin=subprocess.PIPE, shell=True) p.communicate(""" - sudo sysctl -w net.core.somaxconn=1024 + sudo sysctl -w net.core.somaxconn=5000 sudo -s ulimit -n 8192 sudo sysctl net.ipv4.tcp_tw_reuse=1 sudo sysctl net.ipv4.tcp_tw_recycle=1 From 7a5c10ae15a8234c202a45081f571eb12dce9a09 Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Sun, 26 May 2013 21:26:35 -0700 Subject: [PATCH 04/11] Retry ActivePerl download to deal with intermittent 404s Sometimes the ActivePerl HTTP server returns a 404 for a URL that should work, so retry it until it works, but up to a max number of retries. And delay a little in between. --- installer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/installer.py b/installer.py index 6c6cd84c99b..cd7df0e82cd 100644 --- a/installer.py +++ b/installer.py @@ -1,5 +1,6 @@ import subprocess import os +import time class Installer: @@ -110,7 +111,17 @@ def __install_server_software(self): # Perl # - self.__run_command("curl http://downloads.activestate.com/ActivePerl/releases/5.16.3.1603/ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746.tar.gz | tar xvz"); + # Sometimes this HTTP server returns 404, so retry a few times until it works, but don't retry forever + tries = 0 + while True: + self.__run_command("curl http://downloads.activestate.com/ActivePerl/releases/5.16.3.1603/ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746.tar.gz | tar xvz"); + if os.path.exists(os.path.join('installs', 'ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746')): + break + tries += 1 + if tries >= 30: + raise Exception('Could not download ActivePerl after many retries') + time.sleep(5) + self.__run_command("sudo ./install.sh --license-accepted --prefix /opt/ActivePerl-5.16 --no-install-html", cwd="ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746", send_yes=True) self.__run_command("curl -L http://cpanmin.us | perl - --sudo App::cpanminus") self.__run_command("cpanm -f -S DBI DBD::mysql Kelp Dancer Mojolicious Kelp::Module::JSON::XS Dancer::Plugin::Database Starman Plack JSON Web::Simple DBD::Pg JSON::XS EV HTTP::Parser::XS Monoceros") From 6c739f97181637738576e9400f161312d03bd93c Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Sun, 26 May 2013 21:37:00 -0700 Subject: [PATCH 05/11] Update vert.x download URL --- installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.py b/installer.py index cd7df0e82cd..c84d843090b 100644 --- a/installer.py +++ b/installer.py @@ -333,7 +333,7 @@ def __install_server_software(self): ############################## # Vert.x ############################## - self.__run_command("curl http://vertx.io/downloads/vert.x-1.3.1.final.tar.gz | tar xvz") + self.__run_command("curl http://vert-x.github.io/vertx-downloads/downloads/vert.x-1.3.1.final.tar.gz | tar xvz") ############################## # Yesod From c31b4dc063aab4af667962ac6689e29f13bd144c Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Wed, 29 May 2013 19:45:52 -0700 Subject: [PATCH 06/11] Disable IIS HTTP logging Match the overall policy of the other framework benchmarks. On my setup, this improves the requests/sec of a test like 'json' by 3-5%. --- installer.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installer.ps1 b/installer.ps1 index 1b03887f382..96f8a5c71fb 100644 --- a/installer.ps1 +++ b/installer.ps1 @@ -23,6 +23,9 @@ Install-WindowsFeature Web-Asp-Net45 $env:Path += ";C:\Windows\system32\inetsrv"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) appcmd set config -section:system.webServer/httpErrors -errorMode:Detailed | Out-Null +# Disable logging +appcmd set config -section:system.webServer/httpLogging -dontLog:True | Out-Null + # URL Rewrite $rewrite_url = "http://download.microsoft.com/download/6/7/D/67D80164-7DD0-48AF-86E3-DE7A182D6815/rewrite_2.0_rtw_x64.msi" $rewrite_local = "$workdir\rewrite_2.0_rtw_x64.msi" From 63505706988ab84391ae40705b287be23761de1d Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Wed, 29 May 2013 20:20:34 -0700 Subject: [PATCH 07/11] Remove unnecessary HTTP response headers This removes: X-AspNetMvc-Version: 4.0 X-AspNet-Version: 4.0.30319 X-Powered-By: ASP.NET I didn't see any performance improvement, but this is probably a good idea in case the extra bytes cause another network packet to be needed, compared to other frameworks. --- aspnet/src/Application.cs | 2 ++ aspnet/src/Web.config | 2 ++ installer.ps1 | 3 +++ 3 files changed, 7 insertions(+) diff --git a/aspnet/src/Application.cs b/aspnet/src/Application.cs index 5535b523748..de5029f15d2 100644 --- a/aspnet/src/Application.cs +++ b/aspnet/src/Application.cs @@ -23,6 +23,8 @@ public void Init(HttpApplication context) private void Start() { + // Remove X-AspNetMvc-Version HTTP response header + MvcHandler.DisableMvcResponseHeader = true; Routes(); Views(); } diff --git a/aspnet/src/Web.config b/aspnet/src/Web.config index d07f4a409f0..81c4a00f614 100644 --- a/aspnet/src/Web.config +++ b/aspnet/src/Web.config @@ -26,6 +26,8 @@ + + diff --git a/installer.ps1 b/installer.ps1 index 96f8a5c71fb..ab94d0fcdcd 100644 --- a/installer.ps1 +++ b/installer.ps1 @@ -26,6 +26,9 @@ appcmd set config -section:system.webServer/httpErrors -errorMode:Detailed | Out # Disable logging appcmd set config -section:system.webServer/httpLogging -dontLog:True | Out-Null +# Remove unnecessary X-Powered-By HTTP response header +appcmd set config -section:system.webServer/httpProtocol /-"customHeaders.[name='X-Powered-By']" | Out-Null + # URL Rewrite $rewrite_url = "http://download.microsoft.com/download/6/7/D/67D80164-7DD0-48AF-86E3-DE7A182D6815/rewrite_2.0_rtw_x64.msi" $rewrite_local = "$workdir\rewrite_2.0_rtw_x64.msi" From 4b4a4bac93190a25ba8913cd5a08eb19b1cbcd75 Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Thu, 30 May 2013 01:30:20 -0700 Subject: [PATCH 08/11] Disable support for direct access of *.cshtml/*.vbhtml to improve perf We don't need to access URLs like this because we use MVC and URL routing. On my setup, this improves the performance of the 'json' test by 13%. --- aspnet/src/Web.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aspnet/src/Web.config b/aspnet/src/Web.config index 81c4a00f614..1528a5dc8be 100644 --- a/aspnet/src/Web.config +++ b/aspnet/src/Web.config @@ -25,6 +25,11 @@ + + + + From 1f08e132b3c0e22d65331c22ee4d81cdd30e5310 Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Thu, 30 May 2013 14:12:18 -0700 Subject: [PATCH 09/11] Revert "Remove unnecessary HTTP response headers" This reverts commit 63505706988ab84391ae40705b287be23761de1d. --- aspnet/src/Application.cs | 2 -- aspnet/src/Web.config | 2 -- installer.ps1 | 3 --- 3 files changed, 7 deletions(-) diff --git a/aspnet/src/Application.cs b/aspnet/src/Application.cs index de5029f15d2..5535b523748 100644 --- a/aspnet/src/Application.cs +++ b/aspnet/src/Application.cs @@ -23,8 +23,6 @@ public void Init(HttpApplication context) private void Start() { - // Remove X-AspNetMvc-Version HTTP response header - MvcHandler.DisableMvcResponseHeader = true; Routes(); Views(); } diff --git a/aspnet/src/Web.config b/aspnet/src/Web.config index 1528a5dc8be..ebd4fea1092 100644 --- a/aspnet/src/Web.config +++ b/aspnet/src/Web.config @@ -31,8 +31,6 @@ - - diff --git a/installer.ps1 b/installer.ps1 index ab94d0fcdcd..96f8a5c71fb 100644 --- a/installer.ps1 +++ b/installer.ps1 @@ -26,9 +26,6 @@ appcmd set config -section:system.webServer/httpErrors -errorMode:Detailed | Out # Disable logging appcmd set config -section:system.webServer/httpLogging -dontLog:True | Out-Null -# Remove unnecessary X-Powered-By HTTP response header -appcmd set config -section:system.webServer/httpProtocol /-"customHeaders.[name='X-Powered-By']" | Out-Null - # URL Rewrite $rewrite_url = "http://download.microsoft.com/download/6/7/D/67D80164-7DD0-48AF-86E3-DE7A182D6815/rewrite_2.0_rtw_x64.msi" $rewrite_local = "$workdir\rewrite_2.0_rtw_x64.msi" From 24e188bbf461c8e8c6f294abecd4ea57fc2a85dc Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Thu, 30 May 2013 14:15:20 -0700 Subject: [PATCH 10/11] Revert "Disable IIS HTTP logging" This reverts commit c31b4dc063aab4af667962ac6689e29f13bd144c. --- installer.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/installer.ps1 b/installer.ps1 index 96f8a5c71fb..1b03887f382 100644 --- a/installer.ps1 +++ b/installer.ps1 @@ -23,9 +23,6 @@ Install-WindowsFeature Web-Asp-Net45 $env:Path += ";C:\Windows\system32\inetsrv"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) appcmd set config -section:system.webServer/httpErrors -errorMode:Detailed | Out-Null -# Disable logging -appcmd set config -section:system.webServer/httpLogging -dontLog:True | Out-Null - # URL Rewrite $rewrite_url = "http://download.microsoft.com/download/6/7/D/67D80164-7DD0-48AF-86E3-DE7A182D6815/rewrite_2.0_rtw_x64.msi" $rewrite_local = "$workdir\rewrite_2.0_rtw_x64.msi" From 938a92b68a738a7686b74bc60ff68f7af5b012f6 Mon Sep 17 00:00:00 2001 From: Malcolm Evershed Date: Tue, 4 Jun 2013 17:28:14 -0700 Subject: [PATCH 11/11] Increase .NET minWorkerThreads to improve CPU utilization MinWorkerThreads controls how many threads the .NET thread pool creates when a new burst of requests come in. Increase this to properly size the thread pool early on and improve CPU utilization, increasing throughput. I saw gains of 5-10% at least. --- aspnet/src/Application.cs | 23 +++++++++++++++++++++++ aspnet/src/Web.config | 5 +++++ 2 files changed, 28 insertions(+) diff --git a/aspnet/src/Application.cs b/aspnet/src/Application.cs index 5535b523748..7b7f557b87b 100644 --- a/aspnet/src/Application.cs +++ b/aspnet/src/Application.cs @@ -1,3 +1,6 @@ +using System; +using System.Configuration; +using System.Threading; using System.Web; using System.Web.Mvc; using System.Web.Routing; @@ -25,6 +28,7 @@ private void Start() { Routes(); Views(); + Threads(); } private void Routes() @@ -57,6 +61,25 @@ private void Views() ViewEngines.Engines.Add(new RazorViewEngine { ViewLocationFormats = new[] { "~/Views/{0}.cshtml" } }); } + private void Threads() + { + // To improve CPU utilization, increase the number of threads that the .NET thread pool expands by when + // a burst of requests come in. We could do this by editing machine.config/system.web/processModel/minWorkerThreads, + // but that seems too global a change, so we do it in code for just our AppPool. More info: + // + // http://support.microsoft.com/kb/821268 + // http://blogs.msdn.com/b/tmarq/archive/2007/07/21/asp-net-thread-usage-on-iis-7-0-and-6-0.aspx + // http://blogs.msdn.com/b/perfworld/archive/2010/01/13/how-can-i-improve-the-performance-of-asp-net-by-adjusting-the-clr-thread-throttling-properties.aspx + + int newMinWorkerThreads = Convert.ToInt32(ConfigurationManager.AppSettings["minWorkerThreadsPerLogicalProcessor"]); + if (newMinWorkerThreads > 0) + { + int minWorkerThreads, minCompletionPortThreads; + ThreadPool.GetMinThreads(out minWorkerThreads, out minCompletionPortThreads); + ThreadPool.SetMinThreads(Environment.ProcessorCount * newMinWorkerThreads, minCompletionPortThreads); + } + } + public void Dispose() { } diff --git a/aspnet/src/Web.config b/aspnet/src/Web.config index ebd4fea1092..94bfc6ff714 100644 --- a/aspnet/src/Web.config +++ b/aspnet/src/Web.config @@ -29,6 +29,11 @@ + +