From 2e460b7958549687a0d29d43b5c05c952b56250c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Fri, 7 Jun 2024 20:20:19 +0100 Subject: [PATCH 01/12] :sparkles: add `--mysql-skip-create-table` and `--mysql-skip-transfer-data` options --- src/sqlite3_to_mysql/cli.py | 16 +++++++++++++++- src/sqlite3_to_mysql/transporter.py | 13 ++++++++++--- src/sqlite3_to_mysql/types.py | 4 ++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/sqlite3_to_mysql/cli.py b/src/sqlite3_to_mysql/cli.py index 612c063..3f2168a 100644 --- a/src/sqlite3_to_mysql/cli.py +++ b/src/sqlite3_to_mysql/cli.py @@ -120,6 +120,8 @@ ) @click.option("--with-rowid", is_flag=True, help="Transfer rowid columns.") @click.option("-c", "--chunk", type=int, default=None, help="Chunk reading/writing SQL records") +@click.option("-K", "--mysql-skip-create-table", is_flag=True, help="Skip creating tables in MySQL.") +@click.option("-J", "--mysql-skip-transfer-data", is_flag=True, help="Skip transferring data to MySQL.") @click.option("-l", "--log-file", type=click.Path(), help="Log file") @click.option("-q", "--quiet", is_flag=True, help="Quiet. Display only errors.") @click.option("--debug", is_flag=True, help="Debug mode. Will throw exceptions.") @@ -146,6 +148,8 @@ def cli( use_fulltext: bool, with_rowid: bool, chunk: int, + mysql_skip_create_table: bool, + mysql_skip_transfer_data: bool, log_file: t.Union[str, "os.PathLike[t.Any]"], quiet: bool, debug: bool, @@ -159,9 +163,17 @@ def cli( ) if mysql_collation not in set(charset_collations): raise click.ClickException( - f"""Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' is not one of {"'" + "', '".join(charset_collations) + "'"}.""" + f"Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' " + f"""is not one of {"'" + "', '".join(charset_collations) + "'"}.""" ) + # check if both mysql_skip_create_table and mysql_skip_transfer_data are True + if mysql_skip_create_table and mysql_skip_transfer_data: + raise click.ClickException( + "Error: Both -K/--mysql-skip-create-table and -J/--mysql-skip-transfer-data are set. " + "There is nothing to do. Exiting..." + ) + SQLite3toMySQL( sqlite_file=sqlite_file, sqlite_tables=sqlite_tables or tuple(), @@ -183,6 +195,8 @@ def cli( use_fulltext=use_fulltext, with_rowid=with_rowid, chunk=chunk, + mysql_create_tables=not mysql_skip_create_table, + mysql_transfer_data=not mysql_skip_transfer_data, log_file=log_file, quiet=quiet, ).transfer() diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index fb50c39..e84a06d 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -130,6 +130,9 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._sqlite_version = self._get_sqlite_version() self._sqlite_table_xinfo_support = check_sqlite_table_xinfo_support(self._sqlite_version) + self._mysql_create_tables = kwargs.get("mysql_create_tables") or True + self._mysql_transfer_data = kwargs.get("mysql_transfer_data") or True + try: _mysql_connection = mysql.connector.connect( user=self._mysql_user, @@ -677,15 +680,19 @@ def transfer(self) -> None: transfer_rowid: bool = self._with_rowid and self._sqlite_table_has_rowid(table["name"]) # create the table - self._create_table(table["name"], transfer_rowid=transfer_rowid) + if self._mysql_create_tables: + self._create_table(table["name"], transfer_rowid=transfer_rowid) # truncate the table on request if self._mysql_truncate_tables: self._truncate_table(table["name"]) # get the size of the data - self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"') - total_records = int(dict(self._sqlite_cur.fetchone())["total_records"]) + if self._mysql_transfer_data: + self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"') + total_records = int(dict(self._sqlite_cur.fetchone())["total_records"]) + else: + total_records = 0 # only continue if there is anything to transfer if total_records > 0: diff --git a/src/sqlite3_to_mysql/types.py b/src/sqlite3_to_mysql/types.py index ab957ce..d6082f4 100644 --- a/src/sqlite3_to_mysql/types.py +++ b/src/sqlite3_to_mysql/types.py @@ -26,7 +26,9 @@ class SQLite3toMySQLParams(tx.TypedDict): log_file: t.Optional[t.Union[str, "os.PathLike[t.Any]"]] mysql_database: t.Optional[str] mysql_integer_type: t.Optional[str] + mysql_create_tables: t.Optional[bool] mysql_truncate_tables: t.Optional[bool] + mysql_transfer_data: t.Optional[bool] mysql_charset: t.Optional[str] mysql_collation: t.Optional[str] ignore_duplicate_keys: t.Optional[bool] @@ -54,7 +56,9 @@ class SQLite3toMySQLAttributes: _log_file: t.Union[str, "os.PathLike[t.Any]"] _mysql_database: str _mysql_insert_method: str + _mysql_create_tables: bool _mysql_truncate_tables: bool + _mysql_transfer_data: bool _mysql_integer_type: str _mysql_string_type: str _mysql_text_type: str From 785818ab09a97a379a879c5091cfc9376f3f4c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 11:56:41 +0100 Subject: [PATCH 02/12] :safety_vest: add _mysql_transfer_data and _mysql_create_tables warning --- src/sqlite3_to_mysql/transporter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index e84a06d..297c193 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -29,7 +29,6 @@ convert_timedelta, unicase_compare, ) - from .mysql_utils import ( MYSQL_BLOB_COLUMN_TYPES, MYSQL_COLUMN_TYPES, @@ -133,6 +132,9 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._mysql_create_tables = kwargs.get("mysql_create_tables") or True self._mysql_transfer_data = kwargs.get("mysql_transfer_data") or True + if not self._mysql_transfer_data and not self._mysql_create_tables: + raise ValueError("Unable to continue without transferring data or creating tables!") + try: _mysql_connection = mysql.connector.connect( user=self._mysql_user, From 5e99c5f16ae36d3e64c52b85c21a90a4b57145e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 12:03:02 +0100 Subject: [PATCH 03/12] :truck: rename CLI params --- src/sqlite3_to_mysql/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sqlite3_to_mysql/cli.py b/src/sqlite3_to_mysql/cli.py index 3f2168a..7391256 100644 --- a/src/sqlite3_to_mysql/cli.py +++ b/src/sqlite3_to_mysql/cli.py @@ -120,7 +120,7 @@ ) @click.option("--with-rowid", is_flag=True, help="Transfer rowid columns.") @click.option("-c", "--chunk", type=int, default=None, help="Chunk reading/writing SQL records") -@click.option("-K", "--mysql-skip-create-table", is_flag=True, help="Skip creating tables in MySQL.") +@click.option("-K", "--mysql-skip-create-tables", is_flag=True, help="Skip creating tables in MySQL.") @click.option("-J", "--mysql-skip-transfer-data", is_flag=True, help="Skip transferring data to MySQL.") @click.option("-l", "--log-file", type=click.Path(), help="Log file") @click.option("-q", "--quiet", is_flag=True, help="Quiet. Display only errors.") @@ -148,7 +148,7 @@ def cli( use_fulltext: bool, with_rowid: bool, chunk: int, - mysql_skip_create_table: bool, + mysql_skip_create_tables: bool, mysql_skip_transfer_data: bool, log_file: t.Union[str, "os.PathLike[t.Any]"], quiet: bool, @@ -168,9 +168,9 @@ def cli( ) # check if both mysql_skip_create_table and mysql_skip_transfer_data are True - if mysql_skip_create_table and mysql_skip_transfer_data: + if mysql_skip_create_tables and mysql_skip_transfer_data: raise click.ClickException( - "Error: Both -K/--mysql-skip-create-table and -J/--mysql-skip-transfer-data are set. " + "Error: Both -K/--mysql-skip-create-tables and -J/--mysql-skip-transfer-data are set. " "There is nothing to do. Exiting..." ) @@ -195,7 +195,7 @@ def cli( use_fulltext=use_fulltext, with_rowid=with_rowid, chunk=chunk, - mysql_create_tables=not mysql_skip_create_table, + mysql_create_tables=not mysql_skip_create_tables, mysql_transfer_data=not mysql_skip_transfer_data, log_file=log_file, quiet=quiet, From b2fbbacd9274620a7a5b7ba13aedcb73c8efe1e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 12:03:17 +0100 Subject: [PATCH 04/12] :white_check_mark: add CLI tests --- tests/func/test_cli.py | 74 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/func/test_cli.py b/tests/func/test_cli.py index 928967e..c94828e 100644 --- a/tests/func/test_cli.py +++ b/tests/func/test_cli.py @@ -211,6 +211,80 @@ def test_invalid_database_port( } ) + def test_mysql_skip_create_tables( + self, + cli_runner: CliRunner, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mysql_database: Engine, + ) -> None: + result: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-create-tables", + ], + ) + assert result.exit_code == 0 + + def test_mysql_skip_transfer_data( + self, + cli_runner: CliRunner, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mysql_database: Engine, + ) -> None: + result: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-transfer-data", + ], + ) + assert result.exit_code == 0 + + def test_mysql_skip_create_tables_and_transfer_data( + self, + cli_runner: CliRunner, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mysql_database: Engine, + ) -> None: + result: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-create-tables", + "--mysql-skip-transfer-data", + ], + ) + assert result.exit_code > 0 + assert ( + "Error: Both -K/--mysql-skip-create-tables and -J/--mysql-skip-transfer-data are set. " + "There is nothing to do. Exiting..." + ) in result.output + @pytest.mark.parametrize( "mysql_integer_type," "mysql_string_type," From a51185df54493f428081d025522fd827536a04a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 12:06:54 +0100 Subject: [PATCH 05/12] :rotating_light: fix isort imports --- src/sqlite3_to_mysql/transporter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index 297c193..3bb5f79 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -29,6 +29,7 @@ convert_timedelta, unicase_compare, ) + from .mysql_utils import ( MYSQL_BLOB_COLUMN_TYPES, MYSQL_COLUMN_TYPES, From 21704157a0f3c83649f4a6ecb8401cd0700e0946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 12:23:19 +0100 Subject: [PATCH 06/12] :memo: update docs --- README.md | 8 +++++--- docs/README.rst | 35 +++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b38f45d..202ce97 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ sqlite3mysql --help ``` Usage: sqlite3mysql [OPTIONS] - Transfer SQLite to MySQL using the provided CLI options. + sqlite3mysql version 2.1.10 Copyright (c) 2018-2024 Klemen Tusar Options: -f, --sqlite-file PATH SQLite3 database file [required] @@ -52,7 +52,7 @@ Options: -h, --mysql-host TEXT MySQL host. Defaults to localhost. -P, --mysql-port INTEGER MySQL port. Defaults to 3306. -S, --skip-ssl Disable MySQL connection encryption. - -i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT] + -i, --mysql-insert-method [DEFAULT|IGNORE|UPDATE] MySQL insert method. DEFAULT will throw errors when encountering duplicate records; UPDATE will update existing rows; IGNORE @@ -64,7 +64,7 @@ Options: to INT(11). --mysql-string-type TEXT MySQL default string field type. Defaults to VARCHAR(255). - --mysql-text-type [MEDIUMTEXT|TEXT|TINYTEXT|LONGTEXT] + --mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT] MySQL default text field type. Defaults to TEXT. --mysql-charset TEXT MySQL database and table character set @@ -75,6 +75,8 @@ Options: not support InnoDB FULLTEXT indexes! --with-rowid Transfer rowid columns. -c, --chunk INTEGER Chunk reading/writing SQL records + -K, --mysql-skip-create-tables Skip creating tables in MySQL. + -J, --mysql-skip-transfer-data Skip transferring data to MySQL. -l, --log-file PATH Log file -q, --quiet Quiet. Display only errors. --debug Debug mode. Will throw exceptions. diff --git a/docs/README.rst b/docs/README.rst index 5f7548a..cfdf8e1 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -23,37 +23,40 @@ Password Options - ``-p, --prompt-mysql-password``: Prompt for MySQL password - ``--mysql-password TEXT``: MySQL password -Table Options -"""""""""""""" +Connection Options +"""""""""""""""""" -- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys. -- ``-X, --without-foreign-keys``: Do not transfer foreign keys. -- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1' -- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data. +- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost. +- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306. +- ``-S, --skip-ssl``: Disable MySQL connection encryption. Transfer Options """""""""""""""" +- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys. +- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data. +- ``-K, --mysql-skip-create-tables``: Skip creating tables in MySQL. - ``-i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]``: MySQL insert method. DEFAULT will throw errors when encountering duplicate records; UPDATE will update existing rows; IGNORE will ignore insert errors. Defaults to IGNORE. +- ``-J, --mysql-skip-transfer-data``: Skip transferring data to MySQL. +- ``--mysql-integer-type TEXT``: MySQL default integer field type. Defaults to INT(11). +- ``--mysql-string-type TEXT``: MySQL default string field type. Defaults to VARCHAR(255). +- ``--mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT]``: MySQL default text field type. Defaults to TEXT. +- ``--mysql-charset TEXT``: MySQL database and table character set. Defaults to utf8mb4. +` ``--mysql-collation TEXT``: MySQL database and table collation +- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes! +- ``-X, --without-foreign-keys``: Do not transfer foreign keys. +- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1' - ``--with-rowid``: Transfer rowid columns. - ``-c, --chunk INTEGER``: Chunk reading/writing SQL records -Connection Options -"""""""""""""""""" - -- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost. -- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306. -- ``-S, --skip-ssl``: Disable MySQL connection encryption. - Other Options -"""""""""""""" +""""""""""""" -- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes! - ``-l, --log-file PATH``: Log file - ``-q, --quiet``: Quiet. Display only errors. - ``--debug``: Debug mode. Will throw exceptions. - ``--version``: Show the version and exit. -- ``--help``: Show this message and exit. +- ``--help``: Show help message and exit. Docker ^^^^^^ From 507b5db9d45c92eac4810bf6af248af8849a98b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 12:42:12 +0100 Subject: [PATCH 07/12] :bug: fix param parsing --- src/sqlite3_to_mysql/transporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index 3bb5f79..4e70a50 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -130,8 +130,8 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._sqlite_version = self._get_sqlite_version() self._sqlite_table_xinfo_support = check_sqlite_table_xinfo_support(self._sqlite_version) - self._mysql_create_tables = kwargs.get("mysql_create_tables") or True - self._mysql_transfer_data = kwargs.get("mysql_transfer_data") or True + self._mysql_create_tables = kwargs.get("mysql_create_tables", True) + self._mysql_transfer_data = kwargs.get("mysql_transfer_data", True) if not self._mysql_transfer_data and not self._mysql_create_tables: raise ValueError("Unable to continue without transferring data or creating tables!") From d06b7a39c454760969415fdaec312a09309caf2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 12:42:22 +0100 Subject: [PATCH 08/12] :white_check_mark: add more tests --- tests/func/sqlite3_to_mysql_test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/func/sqlite3_to_mysql_test.py b/tests/func/sqlite3_to_mysql_test.py index 28f1fde..eb8f831 100644 --- a/tests/func/sqlite3_to_mysql_test.py +++ b/tests/func/sqlite3_to_mysql_test.py @@ -200,6 +200,34 @@ def test_bad_mysql_connection( ) assert "Unable to connect to MySQL" in str(excinfo.value) + @pytest.mark.init + @pytest.mark.parametrize("quiet", [False, True]) + def test_mysql_skip_create_tables_and_transfer_data( + self, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mocker: MockFixture, + quiet: bool, + ) -> None: + mocker.patch.object( + SQLite3toMySQL, + "transfer", + return_value=None, + ) + with pytest.raises(ValueError) as excinfo: + SQLite3toMySQL( # type: ignore[call-arg] + sqlite_file=sqlite_database, + mysql_user=mysql_credentials.user, + mysql_password=mysql_credentials.password, + mysql_host=mysql_credentials.host, + mysql_port=mysql_credentials.port, + mysql_database=mysql_credentials.database, + mysql_create_tables=False, + mysql_transfer_data=False, + quiet=quiet, + ) + assert "Unable to continue without transferring data or creating tables!" in str(excinfo.value) + @pytest.mark.xfail @pytest.mark.init @pytest.mark.parametrize("quiet", [False, True]) From 88783a440541f39c19d3940232bfe6f82fc58042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 12:50:21 +0100 Subject: [PATCH 09/12] :bug: fix parameter default parsing --- src/sqlite3_to_mysql/transporter.py | 36 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index 4e70a50..7b13123 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -69,39 +69,39 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._mysql_password = str(kwargs.get("mysql_password")) or None - self._mysql_host = kwargs.get("mysql_host") or "localhost" + self._mysql_host = kwargs.get("mysql_host", "localhost") - self._mysql_port = kwargs.get("mysql_port") or 3306 + self._mysql_port = kwargs.get("mysql_port", 3306) - self._sqlite_tables = kwargs.get("sqlite_tables") or tuple() + self._sqlite_tables = kwargs.get("sqlite_tables", tuple()) - self._without_foreign_keys = len(self._sqlite_tables) > 0 or kwargs.get("without_foreign_keys") or False + self._without_foreign_keys = len(self._sqlite_tables) > 0 or kwargs.get("without_foreign_keys", False) - self._mysql_ssl_disabled = kwargs.get("mysql_ssl_disabled") or False + self._mysql_ssl_disabled = kwargs.get("mysql_ssl_disabled", False) - self._chunk_size = kwargs.get("chunk") or None + self._chunk_size = kwargs.get("chunk", None) - self._quiet = kwargs.get("quiet") or False + self._quiet = kwargs.get("quiet", False) - self._logger = self._setup_logger(log_file=kwargs.get("log_file") or None, quiet=self._quiet) + self._logger = self._setup_logger(log_file=kwargs.get("log_file", None), quiet=self._quiet) - self._mysql_database = kwargs.get("mysql_database") or "transfer" + self._mysql_database = kwargs.get("mysql_database", "transfer") - self._mysql_insert_method = str(kwargs.get("mysql_integer_type") or "IGNORE").upper() + self._mysql_insert_method = str(kwargs.get("mysql_integer_type", "IGNORE")).upper() if self._mysql_insert_method not in MYSQL_INSERT_METHOD: self._mysql_insert_method = "IGNORE" - self._mysql_truncate_tables = kwargs.get("mysql_truncate_tables") or False + self._mysql_truncate_tables = kwargs.get("mysql_truncate_tables", False) - self._mysql_integer_type = str(kwargs.get("mysql_integer_type") or "INT(11)").upper() + self._mysql_integer_type = str(kwargs.get("mysql_integer_type", "INT(11)")).upper() - self._mysql_string_type = str(kwargs.get("mysql_string_type") or "VARCHAR(255)").upper() + self._mysql_string_type = str(kwargs.get("mysql_string_type", "VARCHAR(255)")).upper() - self._mysql_text_type = str(kwargs.get("mysql_text_type") or "TEXT").upper() + self._mysql_text_type = str(kwargs.get("mysql_text_type", "TEXT")).upper() if self._mysql_text_type not in MYSQL_TEXT_COLUMN_TYPES: self._mysql_text_type = "TEXT" - self._mysql_charset = kwargs.get("mysql_charset") or "utf8mb4" + self._mysql_charset = kwargs.get("mysql_charset", "utf8mb4") self._mysql_collation = ( kwargs.get("mysql_collation") or CharacterSet().get_default_collation(self._mysql_charset.lower())[0] @@ -109,11 +109,11 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): if not kwargs.get("mysql_collation") and self._mysql_collation == "utf8mb4_0900_ai_ci": self._mysql_collation = "utf8mb4_general_ci" - self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys") or False + self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) - self._use_fulltext = kwargs.get("use_fulltext") or False + self._use_fulltext = kwargs.get("use_fulltext", False) - self._with_rowid = kwargs.get("with_rowid") or False + self._with_rowid = kwargs.get("with_rowid", False) sqlite3.register_adapter(Decimal, adapt_decimal) sqlite3.register_converter("DECIMAL", convert_decimal) From 7b3d2fecfdd878f098d4d1ddee6d793c47a08733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 13:01:51 +0100 Subject: [PATCH 10/12] :bug: fix only transferring data --- src/sqlite3_to_mysql/transporter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index 7b13123..6d2ba4a 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -748,10 +748,11 @@ def transfer(self) -> None: raise # add indices - self._add_indices(table["name"]) + if self._mysql_create_tables: + self._add_indices(table["name"]) # add foreign keys - if not self._without_foreign_keys: + if self._mysql_create_tables and not self._without_foreign_keys: self._add_foreign_keys(table["name"]) except Exception: # pylint: disable=W0706 raise From 5c5b2486b6b077a63d86a073747031f1b0d36e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 13:02:01 +0100 Subject: [PATCH 11/12] :white_check_mark: add more tests --- tests/func/test_cli.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/func/test_cli.py b/tests/func/test_cli.py index c94828e..c6d7d11 100644 --- a/tests/func/test_cli.py +++ b/tests/func/test_cli.py @@ -211,7 +211,7 @@ def test_invalid_database_port( } ) - def test_mysql_skip_create_tables( + def test_mysql_skip_transfer_data( self, cli_runner: CliRunner, sqlite_database: str, @@ -229,19 +229,20 @@ def test_mysql_skip_create_tables( mysql_credentials.user, "--mysql-password", mysql_credentials.password, - "--mysql-skip-create-tables", + "--mysql-skip-transfer-data", ], ) assert result.exit_code == 0 - def test_mysql_skip_transfer_data( + def test_mysql_skip_create_tables( self, cli_runner: CliRunner, sqlite_database: str, mysql_credentials: MySQLCredentials, mysql_database: Engine, ) -> None: - result: Result = cli_runner.invoke( + # First we need to create the tables in the MySQL database + result1: Result = cli_runner.invoke( sqlite3mysql, [ "-f", @@ -255,7 +256,23 @@ def test_mysql_skip_transfer_data( "--mysql-skip-transfer-data", ], ) - assert result.exit_code == 0 + assert result1.exit_code == 0 + + result2: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-create-tables", + ], + ) + assert result2.exit_code == 0 def test_mysql_skip_create_tables_and_transfer_data( self, From cfdf2afd5979f28bf6399543dfcaa068993f6ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klemen=20Tu=C5=A1ar?= Date: Sat, 8 Jun 2024 13:12:50 +0100 Subject: [PATCH 12/12] :rotating_light: fix linter errors --- src/sqlite3_to_mysql/transporter.py | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index 6d2ba4a..fd2e492 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -69,29 +69,29 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._mysql_password = str(kwargs.get("mysql_password")) or None - self._mysql_host = kwargs.get("mysql_host", "localhost") + self._mysql_host = str(kwargs.get("mysql_host", "localhost")) - self._mysql_port = kwargs.get("mysql_port", 3306) + self._mysql_port = kwargs.get("mysql_port", 3306) or 3306 - self._sqlite_tables = kwargs.get("sqlite_tables", tuple()) + self._sqlite_tables = kwargs.get("sqlite_tables") or tuple() - self._without_foreign_keys = len(self._sqlite_tables) > 0 or kwargs.get("without_foreign_keys", False) + self._without_foreign_keys = bool(self._sqlite_tables) or bool(kwargs.get("without_foreign_keys", False)) - self._mysql_ssl_disabled = kwargs.get("mysql_ssl_disabled", False) + self._mysql_ssl_disabled = bool(kwargs.get("mysql_ssl_disabled", False)) - self._chunk_size = kwargs.get("chunk", None) + self._chunk_size = bool(kwargs.get("chunk")) - self._quiet = kwargs.get("quiet", False) + self._quiet = bool(kwargs.get("quiet", False)) self._logger = self._setup_logger(log_file=kwargs.get("log_file", None), quiet=self._quiet) - self._mysql_database = kwargs.get("mysql_database", "transfer") + self._mysql_database = kwargs.get("mysql_database", "transfer") or "transfer" self._mysql_insert_method = str(kwargs.get("mysql_integer_type", "IGNORE")).upper() if self._mysql_insert_method not in MYSQL_INSERT_METHOD: self._mysql_insert_method = "IGNORE" - self._mysql_truncate_tables = kwargs.get("mysql_truncate_tables", False) + self._mysql_truncate_tables = bool(kwargs.get("mysql_truncate_tables", False)) self._mysql_integer_type = str(kwargs.get("mysql_integer_type", "INT(11)")).upper() @@ -101,7 +101,7 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): if self._mysql_text_type not in MYSQL_TEXT_COLUMN_TYPES: self._mysql_text_type = "TEXT" - self._mysql_charset = kwargs.get("mysql_charset", "utf8mb4") + self._mysql_charset = kwargs.get("mysql_charset", "utf8mb4") or "utf8mb4" self._mysql_collation = ( kwargs.get("mysql_collation") or CharacterSet().get_default_collation(self._mysql_charset.lower())[0] @@ -109,11 +109,11 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): if not kwargs.get("mysql_collation") and self._mysql_collation == "utf8mb4_0900_ai_ci": self._mysql_collation = "utf8mb4_general_ci" - self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) + self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) or False - self._use_fulltext = kwargs.get("use_fulltext", False) + self._use_fulltext = kwargs.get("use_fulltext", False) or False - self._with_rowid = kwargs.get("with_rowid", False) + self._with_rowid = kwargs.get("with_rowid", False) or False sqlite3.register_adapter(Decimal, adapt_decimal) sqlite3.register_converter("DECIMAL", convert_decimal) @@ -130,8 +130,8 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._sqlite_version = self._get_sqlite_version() self._sqlite_table_xinfo_support = check_sqlite_table_xinfo_support(self._sqlite_version) - self._mysql_create_tables = kwargs.get("mysql_create_tables", True) - self._mysql_transfer_data = kwargs.get("mysql_transfer_data", True) + self._mysql_create_tables = bool(kwargs.get("mysql_create_tables", True)) + self._mysql_transfer_data = bool(kwargs.get("mysql_transfer_data", True)) if not self._mysql_transfer_data and not self._mysql_create_tables: raise ValueError("Unable to continue without transferring data or creating tables!")