Merge #354

354: Add an option to use index swap in import command r=norkunas a=thislg

# Pull Request

## Related issue
Fixes #277

## What does this PR do?
- This PR adds an option to use index swap to prevent downtime while importing data in an existing index.

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?


Co-authored-by: Thibaut Selingue <[email protected]>

Author: meili-bors[bot]

src/Command/MeilisearchImportCommand.php

@@ -21,6 +21,8 @@
 
 final class MeilisearchImportCommand extends IndexCommand
 {
+    private const TEMP_INDEX_PREFIX = '_tmp_';
+
     private Client $searchClient;
     private ManagerRegistry $managerRegistry;
     private SettingsUpdater $settingsUpdater;
@@ -73,6 +75,12 @@ protected function configure(): void
                 'Timeout (in ms) to get response from the search engine',
                 self::DEFAULT_RESPONSE_TIMEOUT
             )
+            ->addOption(
+                'swap-indices',
+                null,
+                InputOption::VALUE_NONE,
+                'Import to temporary indices and use index swap to prevent downtime'
+            )
         ;
     }
 
@@ -83,6 +91,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $indexes = $this->getEntitiesFromArgs($input, $output);
         $entitiesToIndex = $this->entitiesToIndex($indexes);
         $config = $this->searchService->getConfiguration();
+        $swapIndices = $input->getOption('swap-indices');
+        $initialPrefix = $config['prefix'] ?? '';
+        $prefix = null;
+
+        if ($swapIndices) {
+            $prefix = self::TEMP_INDEX_PREFIX;
+            $config['prefix'] = $prefix.($config['prefix'] ?? '');
+        }
+
         $updateSettings = $input->getOption('update-settings');
         $batchSize = $input->getOption('batch-size') ?? '';
         $batchSize = ctype_digit($batchSize) ? (int) $batchSize : $config->get('batchSize');
@@ -149,10 +166,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $manager->clear();
 
             if ($updateSettings) {
-                $this->settingsUpdater->update($index['prefixed_name'], $responseTimeout);
+                $this->settingsUpdater->update($index['prefixed_name'], $responseTimeout, $prefix ? $prefix.$index['prefixed_name'] : null);
             }
         }
 
+        if ($swapIndices) {
+            $this->swapIndices($indexes, $prefix, $output);
+
+            $config['prefix'] = $initialPrefix;
+        }
+
         $output->writeln('<info>Done!</info>');
 
         return 0;
@@ -210,4 +233,32 @@ private function entitiesToIndex($indexes): array
 
         return array_unique($indexes->all(), SORT_REGULAR);
     }
+
+    private function swapIndices(Collection $indexes, string $prefix, OutputInterface $output): void
+    {
+        $indexPairs = [];
+
+        foreach ($indexes as $index) {
+            $tempIndex = $index;
+            $tempIndex['name'] = $prefix.$tempIndex['name'];
+            $pair = [$tempIndex['name'], $index['name']];
+
+            // Indexes must be declared only once during a swap
+            if (!\in_array($pair, $indexPairs, true)) {
+                $indexPairs[] = $pair;
+            }
+        }
+
+        // swap indexes
+        $output->writeln('<info>Swapping indices...</info>');
+        $this->searchClient->swapIndexes($indexPairs);
+        $output->writeln('<info>Indices swapped.</info>');
+        $output->writeln('<info>Deleting temporary indices...</info>');
+
+        // delete temp indexes
+        foreach ($indexPairs as $pair) {
+            $this->searchService->deleteByIndexName($pair[0]);
+            $output->writeln('<info>Deleted '.$pair[0].'</info>');
+        }
+    }
 }

src/Services/SettingsUpdater.php

@@ -33,7 +33,7 @@ public function __construct(SearchService $searchService, Client $searchClient,
      * @param non-empty-string  $indice
      * @param positive-int|null $responseTimeout
      */
-    public function update(string $indice, ?int $responseTimeout = null): void
+    public function update(string $indice, ?int $responseTimeout = null, ?string $prefixedName = null): void
     {
         $index = (new Collection($this->configuration->get('indices')))->firstWhere('prefixed_name', $indice);
 
@@ -45,7 +45,7 @@ public function update(string $indice, ?int $responseTimeout = null): void
             return;
         }
 
-        $indexName = $index['prefixed_name'];
+        $indexName = $prefixedName ?? $index['prefixed_name'];
         $indexInstance = $this->searchClient->index($indexName);
         $responseTimeout = $responseTimeout ?? self::DEFAULT_RESPONSE_TIMEOUT;
 

tests/Integration/Command/MeilisearchImportCommandTest.php

@@ -391,4 +391,27 @@ public function testAlias(): void
 
         self::assertSame(['meili:import'], $command->getAliases());
     }
+
+    public function testImportingIndexWithSwap(): void
+    {
+        for ($i = 0; $i <= 5; ++$i) {
+            $this->createPost();
+        }
+
+        $command = $this->application->find('meilisearch:import');
+        $commandTester = new CommandTester($command);
+        $return = $commandTester->execute([
+            '--swap-indices' => true,
+        ]);
+
+        $output = $commandTester->getDisplay();
+        $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
+        $this->assertStringContainsString('Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into _tmp_sf_phpunit__'.self::$indexName.' index (6 indexed since start)', $output);
+        $this->assertStringContainsString('Swapping indices...', $output);
+        $this->assertStringContainsString('Indices swapped.', $output);
+        $this->assertStringContainsString('Deleting temporary indices...', $output);
+        $this->assertStringContainsString('Deleted _tmp_sf_phpunit__'.self::$indexName, $output);
+        $this->assertStringContainsString('Done!', $output);
+        $this->assertSame(0, $return);
+    }
 }