Skip to content

Commit 2600cd8

Browse files
laurentjlaurentj
authored
Merge pull request #6735 from 3liz/fix-sqlite-migration
Fix sqlite migration
2 parents 179263e + 61139fb commit 2600cd8

5 files changed

Lines changed: 159 additions & 102 deletions

File tree

‎lizmap/modules/lizmap/lib/App/AbstractMigratorFromSqlite.php‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
<?php
2+
3+
/**
4+
* @author 3liz
5+
* @copyright 2019-2026 3liz
6+
*
7+
* @see https://3liz.com
8+
*
9+
* @license Mozilla Public License : http://www.mozilla.org/MPL/
10+
*/
11+
12+
namespace Lizmap\App;
13+
14+
abstract class AbstractMigratorFromSqlite
15+
{
16+
public function __construct() {}
17+
18+
public const MIGRATE_RES_OK = 1;
19+
public const MIGRATE_RES_ALREADY_MIGRATED = 2;
20+
21+
protected $oldProfile = '';
22+
protected $newProfile = '';
23+
protected $tablesWereReseted = true;
24+
25+
protected function prepareTablesCopy($oldProfile, $newProfile, $tablesWereReseted = true)
26+
{
27+
$this->newProfile = $newProfile;
28+
$this->oldProfile = $oldProfile;
29+
$this->tablesWereReseted = $tablesWereReseted;
30+
}
31+
32+
protected function copyTable($daoSelector, $updateSequence = true, $forceUpdateFields = array(), $updateExisting = false)
33+
{
34+
$daoNew = \jDao::get($daoSelector, $this->newProfile);
35+
$daoSqlite = \jDao::create($daoSelector, $this->oldProfile);
36+
$properties = array_keys($daoSqlite->getProperties());
37+
38+
/** @var \jDaoRecordBase $rec */
39+
foreach ($daoSqlite->findAll() as $rec) {
40+
$daoRec = \jDao::createRecord($daoSelector, $this->newProfile);
41+
foreach ($properties as $prop) {
42+
$daoRec->{$prop} = $rec->{$prop};
43+
}
44+
45+
try {
46+
47+
if (!$this->tablesWereReseted) {
48+
// if content was not deleted before import, we should check if the record is here.
49+
$existingRec = $daoNew->get($rec->getPk());
50+
if ($existingRec) {
51+
if ($updateExisting) {
52+
$daoNew->update($daoRec);
53+
}
54+
55+
continue;
56+
}
57+
}
58+
59+
$daoNew->insert($daoRec);
60+
61+
if (count($forceUpdateFields)) {
62+
$newConn = \jDb::getConnection($this->newProfile);
63+
// $daoNew->insert does not save fields for which there is an "insertpattern" into the dao.
64+
// so we must set the value ourselves.
65+
$tableName = $daoNew->getTables()[$daoNew->getPrimaryTable()]['realname'];
66+
$pkNames = $daoNew->getPrimaryKeyNames();
67+
$pkFieldNames = array();
68+
foreach ($pkNames as $pkName) {
69+
$pkFieldNames[] = $daoNew->getProperties()[$pkName]['fieldName'];
70+
}
71+
72+
$sets = array();
73+
foreach ($forceUpdateFields as $prop) {
74+
if ($rec->{$prop} == '' && !$daoNew->getProperties()[$prop]['required']) {
75+
$val = 'NULL';
76+
} else {
77+
$val = $newConn->quote($rec->{$prop});
78+
}
79+
$fieldName = $daoNew->getProperties()[$prop]['fieldName'];
80+
$sets[] = $newConn->encloseName($fieldName).' = '.$val;
81+
}
82+
$sql = 'UPDATE '.$newConn->prefixTable($tableName).' SET '.implode(',', $sets).' WHERE ';
83+
$pkValues = array_combine($pkFieldNames, is_array($rec->getPk()) ? $rec->getPk() : array($rec->getPk()));
84+
$sqlPk = array();
85+
foreach ($pkValues as $f => $v) {
86+
$sqlPk[] = $newConn->encloseName($f).'='.$newConn->quote($v);
87+
}
88+
$sql .= implode(' AND ', $sqlPk);
89+
$newConn->exec($sql);
90+
}
91+
} catch (\Exception $e) {
92+
echo '*** Insert ERROR for the record ';
93+
var_export($rec->getPk());
94+
echo "\nError is: ".$e->getMessage()."\n";
95+
}
96+
}
97+
98+
if ($updateSequence) {
99+
$idField = $daoNew->getProperties()[$daoNew->getPrimaryKeyNames()[0]]['fieldName'];
100+
$table = $daoNew->getTables()[$daoNew->getPrimaryTable()]['realname'];
101+
102+
$conn = \jDb::getConnection($this->newProfile);
103+
$rs = $conn->query('SELECT pg_get_serial_sequence('.$conn->quote($table).','.$conn->quote($idField).') as sequence_name');
104+
if ($rs && ($rec = $rs->fetch())) {
105+
$sequence = $rec->sequence_name;
106+
if ($sequence) {
107+
$conn->query('SELECT setval('.$conn->quote($sequence).',
108+
(SELECT max('.$conn->encloseName($idField).')
109+
FROM '.$conn->encloseName($table).'))');
110+
}
111+
}
112+
}
113+
}
114+
}

‎lizmap/modules/lizmap/lib/Commands/DbMigrateUsers.php‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,28 +18,29 @@ protected function configure()
1818
->setDescription('Migrate users data from a sqlite database to the current database (experimental)')
1919
->setHelp('')
2020
->addOption('resetbefore', null, InputOption::VALUE_NONE, 'Delete target db before migrating')
21+
->addOption('force', null, InputOption::VALUE_NONE, 'Force migration if there are already some data')
2122
;
2223
}
2324

2425
protected function execute(InputInterface $input, OutputInterface $output)
2526
{
26-
$logMigrator = new MigratorFromSqlite();
27+
$usersMigrator = new MigratorFromSqlite();
2728

2829
try {
29-
$res = $logMigrator->migrateUsersAndRights($input->getOption('resetbefore'));
30+
$res = $usersMigrator->migrateUsersAndRights($input->getOption('resetbefore'), $input->getOption('force'));
3031
} catch (\UnexpectedValueException $e) {
3132
$output->writeln('Error during the migration: '.$e->getMessage());
3233

3334
return 1;
3435
}
3536

3637
switch ($res) {
37-
case $logMigrator::MIGRATE_RES_ALREADY_MIGRATED:
38+
case $usersMigrator::MIGRATE_RES_ALREADY_MIGRATED:
3839
$output->writeln('It seems already migrated, there are some data into existing users tables');
3940

4041
break;
4142

42-
case $logMigrator::MIGRATE_RES_OK:
43+
case $usersMigrator::MIGRATE_RES_OK:
4344
$output->writeln('Migration done');
4445

4546
break;

‎lizmap/modules/lizmap/lib/Logger/MigratorFromSqlite.php‎

Lines changed: 7 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
/**
44
* @author 3liz
5-
* @copyright 2019 3liz
5+
* @copyright 2019-2026 3liz
66
*
77
* @see http://3liz.com
88
*
@@ -11,50 +11,10 @@
1111

1212
namespace Lizmap\Logger;
1313

14-
class MigratorFromSqlite
15-
{
16-
public function __construct() {}
17-
18-
public const MIGRATE_RES_OK = 1;
19-
public const MIGRATE_RES_ALREADY_MIGRATED = 2;
20-
21-
protected function copyTable($daoSelector, $oldProfile, $newProfile, $updateSequence = true)
22-
{
23-
$daoNew = \jDao::get($daoSelector, $newProfile);
24-
$daoSqlite = \jDao::create($daoSelector, $oldProfile);
25-
$properties = array_keys($daoSqlite->getProperties());
26-
foreach ($daoSqlite->findAll() as $rec) {
27-
$daoRec = \jDao::createRecord($daoSelector, $newProfile);
28-
foreach ($properties as $prop) {
29-
$daoRec->{$prop} = $rec->{$prop};
30-
}
31-
32-
try {
33-
$daoNew->insert($daoRec);
34-
} catch (\Exception $e) {
35-
echo '*** Insert ERROR for the record ';
36-
var_export($rec->getPk());
37-
echo "\nError is: ".$e->getMessage()."\n";
38-
}
39-
}
40-
41-
if ($updateSequence) {
42-
$idField = $daoNew->getProperties()[$daoNew->getPrimaryKeyNames()[0]]['fieldName'];
43-
$table = $daoNew->getTables()[$daoNew->getPrimaryTable()]['realname'];
44-
45-
$conn = \jDb::getConnection($newProfile);
46-
$rs = $conn->query('SELECT pg_get_serial_sequence('.$conn->quote($table).','.$conn->quote($idField).') as sequence_name');
47-
if ($rs && ($rec = $rs->fetch())) {
48-
$sequence = $rec->sequence_name;
49-
if ($sequence) {
50-
$conn->query('SELECT setval('.$conn->quote($sequence).',
51-
(SELECT max('.$conn->encloseName($idField).')
52-
FROM '.$conn->encloseName($table).'))');
53-
}
54-
}
55-
}
56-
}
14+
use Lizmap\App\AbstractMigratorFromSqlite;
5715

16+
class MigratorFromSqlite extends AbstractMigratorFromSqlite
17+
{
5818
public function migrateLog($profileName = 'lizlog', $resetBefore = false)
5919
{
6020
$profile = \jProfiles::get('jdb', $profileName);
@@ -93,8 +53,9 @@ public function migrateLog($profileName = 'lizlog', $resetBefore = false)
9353
return self::MIGRATE_RES_ALREADY_MIGRATED;
9454
}
9555

96-
$this->copyTable('lizmap~logCounter', 'oldlizlog', $profileName);
97-
$this->copyTable('lizmap~logDetail', 'oldlizlog', $profileName);
56+
$this->prepareTablesCopy('oldlizlog', $profileName, $resetBefore);
57+
$this->copyTable('lizmap~logCounter');
58+
$this->copyTable('lizmap~logDetail', true, array('log_timestamp'));
9859

9960
return self::MIGRATE_RES_OK;
10061
}

‎lizmap/modules/lizmap/lib/Users/MigratorFromSqlite.php‎

Lines changed: 30 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
/**
44
* @author 3liz
5-
* @copyright 2019 3liz
5+
* @copyright 2019-2026 3liz
66
*
77
* @see http://3liz.com
88
*
@@ -11,14 +11,11 @@
1111

1212
namespace Lizmap\Users;
1313

14-
class MigratorFromSqlite
15-
{
16-
public function __construct() {}
17-
18-
public const MIGRATE_RES_OK = 1;
19-
public const MIGRATE_RES_ALREADY_MIGRATED = 2;
14+
use Lizmap\App\AbstractMigratorFromSqlite;
2015

21-
public function migrateUsersAndRights($resetBefore = false)
16+
class MigratorFromSqlite extends AbstractMigratorFromSqlite
17+
{
18+
public function migrateUsersAndRights($resetBefore = false, $forceMigration = false)
2219
{
2320
$sqliteFile = \jApp::varPath('db/jauth.db');
2421
if (!file_exists($sqliteFile)) {
@@ -50,55 +47,39 @@ public function migrateUsersAndRights($resetBefore = false)
5047
$db->exec('DELETE FROM '.$db->prefixTable('jacl2_group'));
5148
$table = $daoUsersNew->getTables()[$daoUsersNew->getPrimaryTable()]['realname'];
5249
$db->exec('DELETE FROM '.$db->prefixTable($table));
53-
} elseif ($daoUsersNew->countAll() > 0 || $daoGeoBkmNew->countAll() > 0) {
50+
} elseif (!$forceMigration && $daoUsersNew->countAll() > 0 || $daoGeoBkmNew->countAll() > 0) {
5451
return self::MIGRATE_RES_ALREADY_MIGRATED;
5552
}
5653

57-
$this->copyTable($daoUserSelector, 'oldjauth', $profile);
58-
$this->copyTable('jacl2db~jacl2group', 'oldjauth', $profile, false);
59-
$this->copyTable('jacl2db~jacl2subjectgroup', 'oldjauth', $profile, false);
60-
$this->copyTable('jacl2db~jacl2subject', 'oldjauth', $profile, false);
61-
$this->copyTable('jacl2db~jacl2usergroup', 'oldjauth', $profile, false);
62-
$this->copyTable('jacl2db~jacl2rights', 'oldjauth', $profile, false);
63-
$this->copyTable('lizmap~geobookmark', 'oldjauth', $profile, true);
54+
$this->prepareTablesCopy('oldjauth', $profile, $resetBefore);
55+
$this->copyTable($daoUserSelector, true, array('create_date'), true);
56+
$this->copyTable('jacl2db~jacl2group', false);
57+
$this->copyTable('jacl2db~jacl2subjectgroup', false);
58+
$this->copyTable('jacl2db~jacl2subject', false);
59+
60+
if (!$resetBefore) {
61+
$this->deleteExistingRightsForImportedGroups();
62+
}
63+
$this->copyTable('jacl2db~jacl2usergroup', false);
64+
$this->copyTable('jacl2db~jacl2rights', false);
65+
$this->copyTable('lizmap~geobookmark', true);
6466

6567
return self::MIGRATE_RES_OK;
6668
}
6769

68-
protected function copyTable($daoSelector, $oldProfile, $newProfile, $updateSequence = true)
70+
protected function deleteExistingRightsForImportedGroups()
6971
{
70-
$daoNew = \jDao::get($daoSelector, $newProfile);
71-
$daoSqlite = \jDao::create($daoSelector, $oldProfile);
72-
$properties = array_keys($daoSqlite->getProperties());
73-
foreach ($daoSqlite->findAll() as $rec) {
74-
$daoRec = \jDao::createRecord($daoSelector, $newProfile);
75-
foreach ($properties as $prop) {
76-
$daoRec->{$prop} = $rec->{$prop};
77-
}
78-
79-
try {
80-
$daoNew->insert($daoRec);
81-
} catch (\Exception $e) {
82-
echo '*** Insert ERROR for the record ';
83-
var_export($rec->getPk());
84-
echo "\nError is: ".$e->getMessage()."\n";
85-
}
72+
$oldDb = \jDb::getConnection($this->oldProfile);
73+
$newDb = \jDb::getConnection($this->newProfile);
74+
75+
$rs = $oldDb->query('SELECT distinct(id_aclgrp) as id_aclgrp2 FROM '.$oldDb->prefixTable('jacl2_rights'));
76+
foreach ($rs as $rec) {
77+
$newDb->exec('DELETE FROM '.$newDb->prefixTable('jacl2_rights').' WHERE id_aclgrp = '.$newDb->quote($rec->id_aclgrp2));
8678
}
8779

88-
if ($updateSequence) {
89-
$idField = $daoNew->getProperties()[$daoNew->getPrimaryKeyNames()[0]]['fieldName'];
90-
$table = $daoNew->getTables()[$daoNew->getPrimaryTable()]['realname'];
91-
92-
$conn = \jDb::getConnection($newProfile);
93-
$rs = $conn->query('SELECT pg_get_serial_sequence('.$conn->quote($table).','.$conn->quote($idField).') as sequence_name');
94-
if ($rs && ($rec = $rs->fetch())) {
95-
$sequence = $rec->sequence_name;
96-
if ($sequence) {
97-
$conn->query('SELECT setval('.$conn->quote($sequence).',
98-
(SELECT max('.$conn->encloseName($idField).')
99-
FROM '.$conn->encloseName($table).'))');
100-
}
101-
}
80+
$rs = $oldDb->query('SELECT distinct(login) as login2 FROM '.$oldDb->prefixTable('jacl2_user_group'));
81+
foreach ($rs as $rec) {
82+
$newDb->exec('DELETE FROM '.$newDb->prefixTable('jacl2_user_group').' WHERE login = '.$newDb->quote($rec->login2));
10283
}
10384
}
10485

@@ -118,7 +99,7 @@ protected function createUsersTables()
11899

119100
$profile = \jProfiles::get('jdb', $profileName);
120101
if (!$profile) {
121-
throw new \UnexpectedValueException("No {$profile} profile defined into profiles.ini.php", 1);
102+
throw new \UnexpectedValueException("No {$profileName} profile defined into profiles.ini.php", 1);
122103
}
123104

124105
if ($profile['driver'] == 'sqlite3') {
@@ -133,7 +114,7 @@ protected function createUsersTables()
133114
return array($daoSelector, $profileName);
134115
}
135116

136-
// the table does not exists, let's create it
117+
// the table does not exist, let's create it
137118
$mapper = new \jDaoDbMapper($profileName);
138119
$mapper->createTableFromDao($daoSelector);
139120

‎phpstan-baseline.neon‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -512,15 +512,15 @@ parameters:
512512
-
513513
message: "#^Cannot access property \\$sequence_name on object\\|true\\.$#"
514514
count: 1
515-
path: lizmap/modules/lizmap/lib/Users/MigratorFromSqlite.php
515+
path: lizmap/modules/lizmap/lib/App/AbstractMigratorFromSqlite.php
516516

517517
-
518518
message: "#^If condition is always true\\.$#"
519519
count: 1
520-
path: lizmap/modules/lizmap/lib/Users/MigratorFromSqlite.php
520+
path: lizmap/modules/lizmap/lib/App/AbstractMigratorFromSqlite.php
521521

522522
-
523-
message: "#^Part \\$profile \\(array\\{\\}\\) of encapsed string cannot be cast to string\\.$#"
523+
message: "#^If condition is always true\\.$#"
524524
count: 1
525525
path: lizmap/modules/lizmap/lib/Users/MigratorFromSqlite.php
526526

0 commit comments

Comments
 (0)