Merge pull request #19 from rfxn/codex/increase-serverlimit-for-process-overlap

Add ServerLimit headroom to prevent scoreboard full errors

Author: rfxn

README.md

@@ -84,8 +84,8 @@ Avg httpd proc size:    10 MB
   [OK] Client denials      No Client denials observed in sampled lines.
   [--] Apache restarts     1 restart events; max 1/day across 1 day(s).
 ------------------------------------------------
-Current vs Proposed (prefork) (current => proposed):
-  ServerLimit           256          => 256
+  Current vs Proposed (prefork) (current => proposed):
+    ServerLimit           256          => 272
   MaxRequestWorkers     256          => 256
   StartServers          5            => 2
   MinSpareServers       5            => 2
@@ -100,7 +100,7 @@ MaxKeepAliveRequests 100
 KeepAliveTimeout 5
 
 <IfModule prefork.c>
-    ServerLimit            256
+    ServerLimit            272
     MaxRequestWorkers      256
     StartServers           2
     MinSpareServers        2
@@ -128,7 +128,7 @@ JSON output mirrors the same data structure for pipelines:
   "apache_budget_pct": 0.35,
   "apache_budget_source": "tier",
   "log_file": "/var/log/apache-smart-tuner.log",
-  "recommended_block": "# BEGIN APACHE_SMART_TUNER\n# Apache Smart Tuner v1.19.1 (Tier: LOW-MID, MPM: prefork)\nTimeout 120\nKeepAlive On\nMaxKeepAliveRequests 100\nKeepAliveTimeout 5\n\n<IfModule prefork.c>\n    ServerLimit            256\n    MaxRequestWorkers      256\n    StartServers           2\n    MinSpareServers        2\n    MaxSpareServers        8\n    MaxConnectionsPerChild 4000\n</IfModule>\n# END APACHE_SMART_TUNER\n",
+  "recommended_block": "# BEGIN APACHE_SMART_TUNER\n# Apache Smart Tuner v1.19.1 (Tier: LOW-MID, MPM: prefork)\nTimeout 120\nKeepAlive On\nMaxKeepAliveRequests 100\nKeepAliveTimeout 5\n\n<IfModule prefork.c>\n    ServerLimit            272\n    MaxRequestWorkers      256\n    StartServers           2\n    MinSpareServers        2\n    MaxSpareServers        8\n    MaxConnectionsPerChild 4000\n</IfModule>\n# END APACHE_SMART_TUNER\n",
   "log_review": {
     "status": "ready",
     "message": "Analyzed last 15000 lines",
@@ -157,7 +157,7 @@ JSON output mirrors the same data structure for pipelines:
     "MaxConnectionsPerChild": "0"
   },
   "recommended": {
-    "ServerLimit": "256",
+    "ServerLimit": "272",
     "MaxRequestWorkers": "256",
     "ThreadsPerChild": "",
     "StartServers": "2",
@@ -170,6 +170,9 @@ JSON output mirrors the same data structure for pipelines:
 }
 ```
 
+## ServerLimit buffer and "scoreboard full" protection
+Apache keeps old children alive long enough to finish in-flight requests during graceful restarts or MaxConnectionsPerChild recycling. If ServerLimit is set to the exact active process count, those lingering children can temporarily exhaust the scoreboard and block Apache from spawning the replacements, resulting in brief "Scoreboard is full" errors and dropped requests. Apache Smart Tuner pads ServerLimit by a small buffer (5% with an 8-slot minimum) above the calculated MaxRequestWorkers/process count so new workers can come online while old ones drain, without raising steady-state concurrency.
+
 ## Safety details
 - **Backups:** Target include files are copied with a `.bk-YYYYmmddHHMMSS` suffix before any write.
 - **Scrub legacy blocks:** Existing `prefork.c`, `worker.c`, `event.c`, and `mpm_*` blocks inside the target include are removed before the new tuner block is written.

apache-tuner

@@ -65,6 +65,10 @@ RESTART_DAY_COUNT=0
 RESTART_MAX_PER_DAY=0
 LOG_SAMPLE_CONTENT=""
 
+# Scoreboard safety defaults
+SERVER_LIMIT_BUFFER_PCT="0.05"
+MIN_SERVER_LIMIT_BUFFER=8
+
 # Common error patterns to highlight during log review
 declare -A COMMON_ERROR_PATTERNS=(
     [segfaults]="segfault|segmentation fault|child process .*dumped core"
@@ -322,6 +326,37 @@ round_down_to_multiple_of_8() {
     fi
 }
 
+round_up_to_multiple_of_8() {
+    local VALUE=$1
+    local MULTIPLE=8
+
+    if [[ -z "$VALUE" || "$VALUE" -le 0 ]]; then
+        echo 0
+        return
+    fi
+
+    local REMAINDER=$(( VALUE % MULTIPLE ))
+    if [[ "$REMAINDER" -eq 0 ]]; then
+        echo "$VALUE"
+    else
+        echo $(( VALUE + MULTIPLE - REMAINDER ))
+    fi
+}
+
+calculate_server_limit_with_buffer() {
+    local BASE_VALUE=$1
+
+    local RAW_BUFFER
+    RAW_BUFFER=$(awk -v target="$BASE_VALUE" -v pct="$SERVER_LIMIT_BUFFER_PCT" 'BEGIN { printf "%.0f", (target * pct) }')
+
+    if [[ "$RAW_BUFFER" -lt "$MIN_SERVER_LIMIT_BUFFER" ]]; then
+        RAW_BUFFER=$MIN_SERVER_LIMIT_BUFFER
+    fi
+
+    local BUFFER=$(round_up_to_multiple_of_8 "$RAW_BUFFER")
+    echo $(( BASE_VALUE + BUFFER ))
+}
+
 # ----------------- Apache layout / env detection -----------------
 
 detect_apache_layout() {
@@ -1070,7 +1105,7 @@ build_recommended_block() {
         (( MAX_WORKERS > MRW_CAP_PREFORK )) && MAX_WORKERS=$MRW_CAP_PREFORK
         (( MAX_WORKERS < 128 )) && MAX_WORKERS=128
 
-        SERVER_LIMIT=$MAX_WORKERS
+        SERVER_LIMIT=$(calculate_server_limit_with_buffer "$MAX_WORKERS")
         MIN_SPARE_SERVERS=$(round_down_to_multiple_of_8 "$MIN_SPARE")
         MAX_SPARE_SERVERS=$(round_down_to_multiple_of_8 "$MAX_SPARE")
         START_SERVERS_ROUNDED=$(round_down_to_multiple_of_8 "$START_SERVERS")
@@ -1115,7 +1150,7 @@ build_recommended_block() {
         MAX_PROCS=$(round_down_to_multiple_of_8 "$MAX_PROCS")
         (( MAX_PROCS < 1 )) && MAX_PROCS=1
         MAX_WORKERS=$(( MAX_PROCS * THREADS_PER_CHILD ))
-        SERVER_LIMIT="$MAX_PROCS"
+        SERVER_LIMIT=$(calculate_server_limit_with_buffer "$MAX_PROCS")
 
         MIN_SPARE_THREADS=$(( MIN_SPARE * THREADS_PER_CHILD ))
         MAX_SPARE_THREADS=$(( MAX_SPARE * THREADS_PER_CHILD ))
@@ -1129,7 +1164,7 @@ build_recommended_block() {
         (( MIN_SPARE_THREADS > MAX_SPARE_THREADS )) && MIN_SPARE_THREADS=$MAX_SPARE_THREADS
 
         MPM_BLOCK="<IfModule mpm_${MPM_TYPE}_module>\n"
-        MPM_BLOCK+="    ServerLimit            $MAX_PROCS\n"
+        MPM_BLOCK+="    ServerLimit            $SERVER_LIMIT\n"
         MPM_BLOCK+="    StartServers           $START_SERVERS_ROUNDED\n"
         MPM_BLOCK+="    MaxRequestWorkers      $MAX_WORKERS\n"
         MPM_BLOCK+="    ThreadsPerChild        $THREADS_PER_CHILD\n"