fix(config): Migrate PyMemcacheCache to ReconnectingMemcache

_unit-test/check-memcached-backend-test.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+source _unit-test/_test_setup.sh
+source install/ensure-files-from-examples.sh
+
+PYMEMCACHE_BACKEND="django.core.cache.backends.memcached.PyMemcacheCache"
+RECONNECTING_MEMCACHE_BACKEND="sentry.cache.backends.reconnectingmemcache.ReconnectingMemcache"
+MEMCACHED_BACKEND="django.core.cache.backends.memcached.MemcachedCache"
+
+assert_contains() {
+  local file="$1"
+  local expected="$2"
+
+  if ! grep -Fq "$expected" "$file"; then
+    echo "Expected $file to contain:"
+    echo "$expected"
+    echo "Actual:"
+    cat "$file"
+    exit 1
+  fi
+}
+
+assert_not_contains() {
+  local file="$1"
+  local unexpected="$2"
+
+  if grep -Fq "$unexpected" "$file"; then
+    echo "Expected $file not to contain:"
+    echo "$unexpected"
+    echo "Actual:"
+    cat "$file"
+    exit 1
+  fi
+}
+
+write_stock_pymemcache_config() {
+  cat >"$SENTRY_CONFIG_PY" <<EOF
+CACHES = {
+    "default": {
+        "BACKEND": "$PYMEMCACHE_BACKEND",
+        "LOCATION": ["memcached:11211"],
+        "TIMEOUT": 3600,
+        "OPTIONS": {"ignore_exc": True},
+    }
+}
+EOF
+}
+
+write_custom_pymemcache_config() {
+  cat >"$SENTRY_CONFIG_PY" <<EOF
+CACHES = {
+    "default": {
+        "BACKEND": "$PYMEMCACHE_BACKEND",
+        "LOCATION": ["memcached:11211"],
+        "TIMEOUT": 3600,
+        "OPTIONS": {"ignore_exc": True, "timeout": 5, "connect_timeout": 3},
+    }
+}
+EOF
+}
+
+write_memcached_config() {
+  cat >"$SENTRY_CONFIG_PY" <<EOF
+CACHES = {
+    "default": {
+        "BACKEND": "$MEMCACHED_BACKEND",
+        "LOCATION": ["memcached:11211"],
+        "TIMEOUT": 3600,
+        "OPTIONS": {"ignore_exc": True},
+    }
+}
+EOF
+}
+
+echo "Test 1 (current example config)"
+export APPLY_AUTOMATIC_CONFIG_UPDATES=1
+source install/check-memcached-backend.sh
+assert_contains "$SENTRY_CONFIG_PY" "$RECONNECTING_MEMCACHE_BACKEND"
+assert_contains "$SENTRY_CONFIG_PY" '"reconnect_age": 300'
+
+echo "Test 2 (stock PyMemcacheCache config)"
+write_stock_pymemcache_config
+source install/check-memcached-backend.sh
+assert_contains "$SENTRY_CONFIG_PY" "$RECONNECTING_MEMCACHE_BACKEND"
+assert_contains "$SENTRY_CONFIG_PY" '"OPTIONS": {"ignore_exc": True, "reconnect_age": 300}'
+assert_not_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
+
+echo "Test 3 (custom PyMemcacheCache options)"
+write_custom_pymemcache_config
+source install/check-memcached-backend.sh
+assert_contains "$SENTRY_CONFIG_PY" "$RECONNECTING_MEMCACHE_BACKEND"
+assert_contains "$SENTRY_CONFIG_PY" 'setdefault("reconnect_age", 300)'
+assert_contains "$SENTRY_CONFIG_PY" 'CACHES["default"].get("OPTIONS") is None'
+assert_contains "$SENTRY_CONFIG_PY" '"timeout": 5'
+assert_not_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
+
+echo "Test 4 (PyMemcacheCache with automatic updates disabled)"
+write_stock_pymemcache_config
+APPLY_AUTOMATIC_CONFIG_UPDATES=0 source install/check-memcached-backend.sh
+assert_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
+
+echo "Test 5 (legacy MemcachedCache config)"
+write_memcached_config
+if (APPLY_AUTOMATIC_CONFIG_UPDATES=1 source install/check-memcached-backend.sh); then
+  echo "Expected check-memcached-backend.sh to fail on legacy MemcachedCache"
+  exit 1
+fi
+
+report_success

install/check-memcached-backend.sh

@@ -1,16 +1,85 @@
 echo "${_group}Checking memcached backend ..."
 
-if grep -q "\.PyMemcacheCache" "$SENTRY_CONFIG_PY"; then
-  echo "PyMemcacheCache found in $SENTRY_CONFIG_PY, gonna assume you're good."
-else
-  if grep -q "\.MemcachedCache" "$SENTRY_CONFIG_PY"; then
-    echo "MemcachedCache found in $SENTRY_CONFIG_PY, you should switch to PyMemcacheCache."
-    echo "See:"
-    echo "  https://develop.sentry.dev/self-hosted/releases/#breaking-changes"
-    exit 1
-  else
-    echo 'Your setup looks weird. Good luck.'
+if grep -q "\.ReconnectingMemcache" "$SENTRY_CONFIG_PY"; then
+  echo "ReconnectingMemcache found in $SENTRY_CONFIG_PY, you're good."
+elif grep -q "\.PyMemcacheCache" "$SENTRY_CONFIG_PY"; then
+  # PyMemcacheCache is not thread-safe under ContextPropagatingThreadPoolExecutor
+  # and causes ingest-monitors / ingest-occurrences to deadlock.
+  # See: https://github.com/getsentry/self-hosted/issues/4301
+
+  apply_config_changes_memcache=0
+  if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then
+    echo
+    echo "PyMemcacheCache is no longer safe to use. The monitor consumer now runs"
+    echo "check-in processing in a ContextPropagatingThreadPoolExecutor, which shares"
+    echo "the non-thread-safe pymemcache client across threads, causing deadlocks."
+    echo
+    echo "We need to swap to ReconnectingMemcache (per-thread clients) in your config."
+    echo "Do you want us to make this change automatically for you?"
+    echo
+
+    yn=""
+    until [ ! -z "$yn" ]; do
+      read -p "y or n? " yn
+      case $yn in
+      y | yes | 1)
+        export apply_config_changes_memcache=1
+        echo
+        echo -n "Thank you."
+        ;;
+      n | no | 0)
+        export apply_config_changes_memcache=0
+        echo
+        echo -n "Alright, you will need to update your sentry.conf.py file manually."
+        echo " See: https://github.com/getsentry/self-hosted/issues/4301"
+        ;;
+      *) yn="" ;;
+      esac
+    done
+
+    echo
+    echo "To avoid this prompt in the future, use one of these flags:"
+    echo
+    echo "  --apply-automatic-config-updates"
+    echo "  --no-apply-automatic-config-updates"
+    echo
+    echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:"
+    echo
+    echo "  APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates"
+    echo "  APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates"
+    echo
+    sleep 5
   fi
+
+  if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_memcache" == 1 ]]; then
+    echo "Migrating $SENTRY_CONFIG_PY to use ReconnectingMemcache"
+    sed -i 's|django\.core\.cache\.backends\.memcached\.PyMemcacheCache|sentry.cache.backends.reconnectingmemcache.ReconnectingMemcache|g' "$SENTRY_CONFIG_PY"
+
+    if ! grep -q "reconnect_age" "$SENTRY_CONFIG_PY"; then
+      sed -i 's/"ignore_exc": True}/"ignore_exc": True, "reconnect_age": 300}/g' "$SENTRY_CONFIG_PY"
+    fi
+
+    if ! grep -q "reconnect_age" "$SENTRY_CONFIG_PY"; then
+      cat <<'EOF' >>"$SENTRY_CONFIG_PY"
+
+# Added by self-hosted install to keep ReconnectingMemcache aligned with
+# the bundled Sentry image configuration.
+if "CACHES" in globals() and "default" in CACHES:
+    if CACHES["default"].get("OPTIONS") is None:
+        CACHES["default"]["OPTIONS"] = {}
+    CACHES["default"]["OPTIONS"].setdefault("reconnect_age", 300)
+EOF
+    fi
+
+    echo "Migrated $SENTRY_CONFIG_PY to use ReconnectingMemcache"
+  fi
+elif grep -q "\.MemcachedCache" "$SENTRY_CONFIG_PY"; then
+  echo "MemcachedCache found in $SENTRY_CONFIG_PY, you should switch to ReconnectingMemcache."
+  echo "See:"
+  echo "  https://develop.sentry.dev/self-hosted/releases/#breaking-changes"
+  exit 1
+else
+  echo 'Your setup looks weird. Good luck.'
 fi
 
 echo "${_endgroup}"

sentry/sentry.conf.example.py

@@ -208,10 +208,10 @@ def get_internal_network():
 
 CACHES = {
     "default": {
-        "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+        "BACKEND": "sentry.cache.backends.reconnectingmemcache.ReconnectingMemcache",
         "LOCATION": ["memcached:11211"],
         "TIMEOUT": 3600,
-        "OPTIONS": {"ignore_exc": True},
+        "OPTIONS": {"ignore_exc": True, "reconnect_age": 300},
     }
 }