Retry connection on socket error (#106)

* WIP retry connection on socket error

currently does not handle fwrite failure, only socket read

* improved implementation

* cleanup

* add test to verify number of retries

Author: cebe

CHANGELOG.md

@@ -7,6 +7,8 @@ Yii Framework 2 redis extension Change Log
 - Bug #114: Fixed ActiveQuery `not between` and `not` conditions which where not working correctly (cebe, ak1987)
 - Bug #123: Fixed ActiveQuery to work with negative limit values, which are used in ActiveDataProvider for the count query (cebe)
 - Enh #9: Added orderBy support to redis ActiveQuery and LuaScriptBuilder (valinurovam)
+- Enh #91: Added option to retry connection after failing to communicate with redis server on stale socket (cebe)
+- Enh #106: Improved handling of connection errors and introduced `yii\redis\SocketException` for these (cebe)
 - Chg #127: Added PHP 7.2 compatibility (brandonkelly)
 
 

Connection.php

@@ -239,12 +239,12 @@ class Connection extends Component
 
     /**
      * @var string the hostname or ip address to use for connecting to the redis server. Defaults to 'localhost'.
-     * If [[unixSocket]] is specified, hostname and port will be ignored.
+     * If [[unixSocket]] is specified, hostname and [[port]] will be ignored.
      */
     public $hostname = 'localhost';
     /**
      * @var integer the port to use for connecting to the redis server. Default port is 6379.
-     * If [[unixSocket]] is specified, hostname and port will be ignored.
+     * If [[unixSocket]] is specified, [[hostname]] and port will be ignored.
      */
     public $port = 6379;
     /**
@@ -279,6 +279,13 @@ class Connection extends Component
      * @since 2.0.5
      */
     public $socketClientFlags = STREAM_CLIENT_CONNECT;
+    /**
+     * @var integer The number of times a command execution should be retried when a connection failure occurs.
+     * This is used in [[executeCommand()]] when a [[SocketException]] is thrown.
+     * Defaults to 0 meaning no retries on failure.
+     * @since 2.0.7
+     */
+    public $retries = 0;
     /**
      * @var array List of available redis commands.
      * @see http://redis.io/commands
@@ -555,7 +562,11 @@ public function close()
         if ($this->_socket !== false) {
             $connection = ($this->unixSocket ?: $this->hostname . ':' . $this->port) . ', database=' . $this->database;
             \Yii::trace('Closing DB connection: ' . $connection, __METHOD__);
-            $this->executeCommand('QUIT');
+            try {
+                $this->executeCommand('QUIT');
+            } catch (SocketException $e) {
+                // ignore errors when quitting a closed connection
+            }
             stream_socket_shutdown($this->_socket, STREAM_SHUT_RDWR);
             $this->_socket = false;
         }
@@ -647,8 +658,38 @@ public function executeCommand($name, $params = [])
         }
 
         \Yii::trace("Executing Redis Command: {$name}", __METHOD__);
-        fwrite($this->_socket, $command);
+        if ($this->retries > 0) {
+            $tries = $this->retries;
+            while ($tries-- > 0) {
+                try {
+                    return $this->sendCommandInternal($command, $params);
+                } catch (SocketException $e) {
+                    \Yii::error($e, __METHOD__);
+                    // backup retries, fail on commands that fail inside here
+                    $retries = $this->retries;
+                    $this->retries = 0;
+                    $this->close();
+                    $this->open();
+                    $this->retries = $retries;
+                }
+            }
+        }
+        return $this->sendCommandInternal($command, $params);
+    }
 
+    /**
+     * Sends RAW command string to the server.
+     * @throws SocketException on connection error.
+     */
+    private function sendCommandInternal($command, $params)
+    {
+        $written = @fwrite($this->_socket, $command);
+        if ($written === false) {
+            throw new SocketException("Failed to write to socket.\nRedis command was: " . $command);
+        }
+        if ($written !== ($len = mb_strlen($command, '8bit'))) {
+            throw new SocketException("Failed to write to socket. $written of $len bytes written.\nRedis command was: " . $command);
+        }
         return $this->parseResponse(implode(' ', $params));
     }
 
@@ -660,7 +701,7 @@ public function executeCommand($name, $params = [])
     private function parseResponse($command)
     {
         if (($line = fgets($this->_socket)) === false) {
-            throw new Exception("Failed to read from socket.\nRedis command was: " . $command);
+            throw new SocketException("Failed to read from socket.\nRedis command was: " . $command);
         }
         $type = $line[0];
         $line = mb_substr($line, 1, -2, '8bit');
@@ -684,7 +725,7 @@ private function parseResponse($command)
                 $data = '';
                 while ($length > 0) {
                     if (($block = fread($this->_socket, $length)) === false) {
-                        throw new Exception("Failed to read from socket.\nRedis command was: " . $command);
+                        throw new SocketException("Failed to read from socket.\nRedis command was: " . $command);
                     }
                     $data .= $block;
                     $length -= mb_strlen($block, '8bit');

SocketException.php

@@ -0,0 +1,25 @@
+<?php
+/**
+ * @link http://www.yiiframework.com/
+ * @copyright Copyright (c) 2008 Yii Software LLC
+ * @license http://www.yiiframework.com/license/
+ */
+
+namespace yii\redis;
+
+use yii\db\Exception;
+
+/**
+ * SocketException indicates a socket connection failure in [[Connection]].
+ * @since 2.0.7
+ */
+class SocketException extends Exception
+{
+    /**
+     * @return string the user-friendly name of this exception
+     */
+    public function getName()
+    {
+        return 'Redis Socket Exception';
+    }
+}

tests/RedisConnectionTest.php

@@ -1,12 +1,23 @@
 <?php
 
 namespace yiiunit\extensions\redis;
+use Yii;
+use yii\helpers\ArrayHelper;
+use yii\log\Logger;
+use yii\redis\Connection;
+use yii\redis\SocketException;
 
 /**
  * @group redis
  */
 class ConnectionTest extends TestCase
 {
+    protected function tearDown()
+    {
+        $this->getConnection(false)->configSet('timeout', 0);
+        parent::tearDown();
+    }
+
     /**
      * test connection to redis and selection of db
      */
@@ -87,6 +98,77 @@ public function testSerialize()
         $this->assertTrue($db2->ping());
     }
 
+    public function testConnectionTimeout()
+    {
+        $db = $this->getConnection(false);
+        $db->configSet('timeout', 1);
+        $this->assertTrue($db->ping());
+        sleep(1);
+        $this->assertTrue($db->ping());
+        sleep(2);
+        if (method_exists($this, 'setExpectedException')) {
+            $this->setExpectedException('\yii\redis\SocketException');
+        } else {
+            $this->expectException('\yii\redis\SocketException');
+        }
+        $this->assertTrue($db->ping());
+    }
+
+    public function testConnectionTimeoutRetry()
+    {
+        $logger = new Logger();
+        Yii::setLogger($logger);
+
+        $db = $this->getConnection(false);
+        $db->retries = 1;
+        $db->configSet('timeout', 1);
+        $this->assertCount(3, $logger->messages, 'log of connection and init commands.');
+
+        $this->assertTrue($db->ping());
+        $this->assertCount(4, $logger->messages, 'log +1 ping command.');
+        usleep(500000); // 500ms
+
+        $this->assertTrue($db->ping());
+        $this->assertCount(5, $logger->messages, 'log +1 ping command.');
+        sleep(2);
+
+        // reconnect should happen here
+
+        $this->assertTrue($db->ping());
+        $this->assertCount(11, $logger->messages, 'log +1 ping command, and reconnection.'
+            . print_r(array_map(function($s) { return (string) $s; }, ArrayHelper::getColumn($logger->messages, 0)), true));
+    }
+
+    /**
+     * Retry connecting 2 times
+     */
+    public function testConnectionTimeoutRetryCount()
+    {
+        $logger = new Logger();
+        Yii::setLogger($logger);
+
+        $db = $this->getConnection(false);
+        $db->retries = 2;
+        $db->configSet('timeout', 1);
+        $db->on(Connection::EVENT_AFTER_OPEN, function() {
+            // sleep 2 seconds after connect to make every command time out
+            sleep(2);
+        });
+        $this->assertCount(3, $logger->messages, 'log of connection and init commands.');
+
+        $exception = false;
+        try {
+            // should try to reconnect 2 times, before finally failing
+            // results in 3 times sending the PING command to redis
+            sleep(2);
+            $db->ping();
+        } catch (SocketException $e) {
+            $exception = true;
+        }
+        $this->assertTrue($exception, 'SocketException should have been thrown.');
+        $this->assertCount(14, $logger->messages, 'log +1 ping command, and reconnection.'
+            . print_r(array_map(function($s) { return (string) $s; }, ArrayHelper::getColumn($logger->messages, 0)), true));
+    }
 
     /**
      * https://github.com/yiisoft/yii2/issues/4745