diff --git a/README.md b/README.md index b38f45d..202ce97 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ sqlite3mysql --help ``` Usage: sqlite3mysql [OPTIONS] - Transfer SQLite to MySQL using the provided CLI options. + sqlite3mysql version 2.1.10 Copyright (c) 2018-2024 Klemen Tusar Options: -f, --sqlite-file PATH SQLite3 database file [required] @@ -52,7 +52,7 @@ Options: -h, --mysql-host TEXT MySQL host. Defaults to localhost. -P, --mysql-port INTEGER MySQL port. Defaults to 3306. -S, --skip-ssl Disable MySQL connection encryption. - -i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT] + -i, --mysql-insert-method [DEFAULT|IGNORE|UPDATE] MySQL insert method. DEFAULT will throw errors when encountering duplicate records; UPDATE will update existing rows; IGNORE @@ -64,7 +64,7 @@ Options: to INT(11). --mysql-string-type TEXT MySQL default string field type. Defaults to VARCHAR(255). - --mysql-text-type [MEDIUMTEXT|TEXT|TINYTEXT|LONGTEXT] + --mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT] MySQL default text field type. Defaults to TEXT. --mysql-charset TEXT MySQL database and table character set @@ -75,6 +75,8 @@ Options: not support InnoDB FULLTEXT indexes! --with-rowid Transfer rowid columns. -c, --chunk INTEGER Chunk reading/writing SQL records + -K, --mysql-skip-create-tables Skip creating tables in MySQL. + -J, --mysql-skip-transfer-data Skip transferring data to MySQL. -l, --log-file PATH Log file -q, --quiet Quiet. Display only errors. --debug Debug mode. Will throw exceptions. diff --git a/docs/README.rst b/docs/README.rst index 5f7548a..cfdf8e1 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -23,37 +23,40 @@ Password Options - ``-p, --prompt-mysql-password``: Prompt for MySQL password - ``--mysql-password TEXT``: MySQL password -Table Options -"""""""""""""" +Connection Options +"""""""""""""""""" -- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys. -- ``-X, --without-foreign-keys``: Do not transfer foreign keys. -- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1' -- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data. +- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost. +- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306. +- ``-S, --skip-ssl``: Disable MySQL connection encryption. Transfer Options """""""""""""""" +- ``-t, --sqlite-tables TUPLE``: Transfer only these specific tables (space separated table names). Implies ``--without-foreign-keys`` which inhibits the transfer of foreign keys. +- ``-E, --mysql-truncate-tables``: Truncates existing tables before inserting data. +- ``-K, --mysql-skip-create-tables``: Skip creating tables in MySQL. - ``-i, --mysql-insert-method [UPDATE|IGNORE|DEFAULT]``: MySQL insert method. DEFAULT will throw errors when encountering duplicate records; UPDATE will update existing rows; IGNORE will ignore insert errors. Defaults to IGNORE. +- ``-J, --mysql-skip-transfer-data``: Skip transferring data to MySQL. +- ``--mysql-integer-type TEXT``: MySQL default integer field type. Defaults to INT(11). +- ``--mysql-string-type TEXT``: MySQL default string field type. Defaults to VARCHAR(255). +- ``--mysql-text-type [LONGTEXT|MEDIUMTEXT|TEXT|TINYTEXT]``: MySQL default text field type. Defaults to TEXT. +- ``--mysql-charset TEXT``: MySQL database and table character set. Defaults to utf8mb4. +` ``--mysql-collation TEXT``: MySQL database and table collation +- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes! +- ``-X, --without-foreign-keys``: Do not transfer foreign keys. +- ``-W, --ignore-duplicate-keys``: Ignore duplicate keys. The default behavior is to create new ones with a numerical suffix, e.g. 'existing_key' -> 'existing_key_1' - ``--with-rowid``: Transfer rowid columns. - ``-c, --chunk INTEGER``: Chunk reading/writing SQL records -Connection Options -"""""""""""""""""" - -- ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost. -- ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306. -- ``-S, --skip-ssl``: Disable MySQL connection encryption. - Other Options -"""""""""""""" +""""""""""""" -- ``-T, --use-fulltext``: Use FULLTEXT indexes on TEXT columns. Will throw an error if your MySQL version does not support InnoDB FULLTEXT indexes! - ``-l, --log-file PATH``: Log file - ``-q, --quiet``: Quiet. Display only errors. - ``--debug``: Debug mode. Will throw exceptions. - ``--version``: Show the version and exit. -- ``--help``: Show this message and exit. +- ``--help``: Show help message and exit. Docker ^^^^^^ diff --git a/src/sqlite3_to_mysql/cli.py b/src/sqlite3_to_mysql/cli.py index 612c063..7391256 100644 --- a/src/sqlite3_to_mysql/cli.py +++ b/src/sqlite3_to_mysql/cli.py @@ -120,6 +120,8 @@ ) @click.option("--with-rowid", is_flag=True, help="Transfer rowid columns.") @click.option("-c", "--chunk", type=int, default=None, help="Chunk reading/writing SQL records") +@click.option("-K", "--mysql-skip-create-tables", is_flag=True, help="Skip creating tables in MySQL.") +@click.option("-J", "--mysql-skip-transfer-data", is_flag=True, help="Skip transferring data to MySQL.") @click.option("-l", "--log-file", type=click.Path(), help="Log file") @click.option("-q", "--quiet", is_flag=True, help="Quiet. Display only errors.") @click.option("--debug", is_flag=True, help="Debug mode. Will throw exceptions.") @@ -146,6 +148,8 @@ def cli( use_fulltext: bool, with_rowid: bool, chunk: int, + mysql_skip_create_tables: bool, + mysql_skip_transfer_data: bool, log_file: t.Union[str, "os.PathLike[t.Any]"], quiet: bool, debug: bool, @@ -159,9 +163,17 @@ def cli( ) if mysql_collation not in set(charset_collations): raise click.ClickException( - f"""Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' is not one of {"'" + "', '".join(charset_collations) + "'"}.""" + f"Error: Invalid value for '--collation' of charset '{mysql_charset}': '{mysql_collation}' " + f"""is not one of {"'" + "', '".join(charset_collations) + "'"}.""" ) + # check if both mysql_skip_create_table and mysql_skip_transfer_data are True + if mysql_skip_create_tables and mysql_skip_transfer_data: + raise click.ClickException( + "Error: Both -K/--mysql-skip-create-tables and -J/--mysql-skip-transfer-data are set. " + "There is nothing to do. Exiting..." + ) + SQLite3toMySQL( sqlite_file=sqlite_file, sqlite_tables=sqlite_tables or tuple(), @@ -183,6 +195,8 @@ def cli( use_fulltext=use_fulltext, with_rowid=with_rowid, chunk=chunk, + mysql_create_tables=not mysql_skip_create_tables, + mysql_transfer_data=not mysql_skip_transfer_data, log_file=log_file, quiet=quiet, ).transfer() diff --git a/src/sqlite3_to_mysql/transporter.py b/src/sqlite3_to_mysql/transporter.py index fb50c39..fd2e492 100644 --- a/src/sqlite3_to_mysql/transporter.py +++ b/src/sqlite3_to_mysql/transporter.py @@ -69,39 +69,39 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._mysql_password = str(kwargs.get("mysql_password")) or None - self._mysql_host = kwargs.get("mysql_host") or "localhost" + self._mysql_host = str(kwargs.get("mysql_host", "localhost")) - self._mysql_port = kwargs.get("mysql_port") or 3306 + self._mysql_port = kwargs.get("mysql_port", 3306) or 3306 self._sqlite_tables = kwargs.get("sqlite_tables") or tuple() - self._without_foreign_keys = len(self._sqlite_tables) > 0 or kwargs.get("without_foreign_keys") or False + self._without_foreign_keys = bool(self._sqlite_tables) or bool(kwargs.get("without_foreign_keys", False)) - self._mysql_ssl_disabled = kwargs.get("mysql_ssl_disabled") or False + self._mysql_ssl_disabled = bool(kwargs.get("mysql_ssl_disabled", False)) - self._chunk_size = kwargs.get("chunk") or None + self._chunk_size = bool(kwargs.get("chunk")) - self._quiet = kwargs.get("quiet") or False + self._quiet = bool(kwargs.get("quiet", False)) - self._logger = self._setup_logger(log_file=kwargs.get("log_file") or None, quiet=self._quiet) + self._logger = self._setup_logger(log_file=kwargs.get("log_file", None), quiet=self._quiet) - self._mysql_database = kwargs.get("mysql_database") or "transfer" + self._mysql_database = kwargs.get("mysql_database", "transfer") or "transfer" - self._mysql_insert_method = str(kwargs.get("mysql_integer_type") or "IGNORE").upper() + self._mysql_insert_method = str(kwargs.get("mysql_integer_type", "IGNORE")).upper() if self._mysql_insert_method not in MYSQL_INSERT_METHOD: self._mysql_insert_method = "IGNORE" - self._mysql_truncate_tables = kwargs.get("mysql_truncate_tables") or False + self._mysql_truncate_tables = bool(kwargs.get("mysql_truncate_tables", False)) - self._mysql_integer_type = str(kwargs.get("mysql_integer_type") or "INT(11)").upper() + self._mysql_integer_type = str(kwargs.get("mysql_integer_type", "INT(11)")).upper() - self._mysql_string_type = str(kwargs.get("mysql_string_type") or "VARCHAR(255)").upper() + self._mysql_string_type = str(kwargs.get("mysql_string_type", "VARCHAR(255)")).upper() - self._mysql_text_type = str(kwargs.get("mysql_text_type") or "TEXT").upper() + self._mysql_text_type = str(kwargs.get("mysql_text_type", "TEXT")).upper() if self._mysql_text_type not in MYSQL_TEXT_COLUMN_TYPES: self._mysql_text_type = "TEXT" - self._mysql_charset = kwargs.get("mysql_charset") or "utf8mb4" + self._mysql_charset = kwargs.get("mysql_charset", "utf8mb4") or "utf8mb4" self._mysql_collation = ( kwargs.get("mysql_collation") or CharacterSet().get_default_collation(self._mysql_charset.lower())[0] @@ -109,11 +109,11 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): if not kwargs.get("mysql_collation") and self._mysql_collation == "utf8mb4_0900_ai_ci": self._mysql_collation = "utf8mb4_general_ci" - self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys") or False + self._ignore_duplicate_keys = kwargs.get("ignore_duplicate_keys", False) or False - self._use_fulltext = kwargs.get("use_fulltext") or False + self._use_fulltext = kwargs.get("use_fulltext", False) or False - self._with_rowid = kwargs.get("with_rowid") or False + self._with_rowid = kwargs.get("with_rowid", False) or False sqlite3.register_adapter(Decimal, adapt_decimal) sqlite3.register_converter("DECIMAL", convert_decimal) @@ -130,6 +130,12 @@ def __init__(self, **kwargs: tx.Unpack[SQLite3toMySQLParams]): self._sqlite_version = self._get_sqlite_version() self._sqlite_table_xinfo_support = check_sqlite_table_xinfo_support(self._sqlite_version) + self._mysql_create_tables = bool(kwargs.get("mysql_create_tables", True)) + self._mysql_transfer_data = bool(kwargs.get("mysql_transfer_data", True)) + + if not self._mysql_transfer_data and not self._mysql_create_tables: + raise ValueError("Unable to continue without transferring data or creating tables!") + try: _mysql_connection = mysql.connector.connect( user=self._mysql_user, @@ -677,15 +683,19 @@ def transfer(self) -> None: transfer_rowid: bool = self._with_rowid and self._sqlite_table_has_rowid(table["name"]) # create the table - self._create_table(table["name"], transfer_rowid=transfer_rowid) + if self._mysql_create_tables: + self._create_table(table["name"], transfer_rowid=transfer_rowid) # truncate the table on request if self._mysql_truncate_tables: self._truncate_table(table["name"]) # get the size of the data - self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"') - total_records = int(dict(self._sqlite_cur.fetchone())["total_records"]) + if self._mysql_transfer_data: + self._sqlite_cur.execute(f'SELECT COUNT(*) AS total_records FROM "{table["name"]}"') + total_records = int(dict(self._sqlite_cur.fetchone())["total_records"]) + else: + total_records = 0 # only continue if there is anything to transfer if total_records > 0: @@ -738,10 +748,11 @@ def transfer(self) -> None: raise # add indices - self._add_indices(table["name"]) + if self._mysql_create_tables: + self._add_indices(table["name"]) # add foreign keys - if not self._without_foreign_keys: + if self._mysql_create_tables and not self._without_foreign_keys: self._add_foreign_keys(table["name"]) except Exception: # pylint: disable=W0706 raise diff --git a/src/sqlite3_to_mysql/types.py b/src/sqlite3_to_mysql/types.py index ab957ce..d6082f4 100644 --- a/src/sqlite3_to_mysql/types.py +++ b/src/sqlite3_to_mysql/types.py @@ -26,7 +26,9 @@ class SQLite3toMySQLParams(tx.TypedDict): log_file: t.Optional[t.Union[str, "os.PathLike[t.Any]"]] mysql_database: t.Optional[str] mysql_integer_type: t.Optional[str] + mysql_create_tables: t.Optional[bool] mysql_truncate_tables: t.Optional[bool] + mysql_transfer_data: t.Optional[bool] mysql_charset: t.Optional[str] mysql_collation: t.Optional[str] ignore_duplicate_keys: t.Optional[bool] @@ -54,7 +56,9 @@ class SQLite3toMySQLAttributes: _log_file: t.Union[str, "os.PathLike[t.Any]"] _mysql_database: str _mysql_insert_method: str + _mysql_create_tables: bool _mysql_truncate_tables: bool + _mysql_transfer_data: bool _mysql_integer_type: str _mysql_string_type: str _mysql_text_type: str diff --git a/tests/func/sqlite3_to_mysql_test.py b/tests/func/sqlite3_to_mysql_test.py index 28f1fde..eb8f831 100644 --- a/tests/func/sqlite3_to_mysql_test.py +++ b/tests/func/sqlite3_to_mysql_test.py @@ -200,6 +200,34 @@ def test_bad_mysql_connection( ) assert "Unable to connect to MySQL" in str(excinfo.value) + @pytest.mark.init + @pytest.mark.parametrize("quiet", [False, True]) + def test_mysql_skip_create_tables_and_transfer_data( + self, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mocker: MockFixture, + quiet: bool, + ) -> None: + mocker.patch.object( + SQLite3toMySQL, + "transfer", + return_value=None, + ) + with pytest.raises(ValueError) as excinfo: + SQLite3toMySQL( # type: ignore[call-arg] + sqlite_file=sqlite_database, + mysql_user=mysql_credentials.user, + mysql_password=mysql_credentials.password, + mysql_host=mysql_credentials.host, + mysql_port=mysql_credentials.port, + mysql_database=mysql_credentials.database, + mysql_create_tables=False, + mysql_transfer_data=False, + quiet=quiet, + ) + assert "Unable to continue without transferring data or creating tables!" in str(excinfo.value) + @pytest.mark.xfail @pytest.mark.init @pytest.mark.parametrize("quiet", [False, True]) diff --git a/tests/func/test_cli.py b/tests/func/test_cli.py index 928967e..c6d7d11 100644 --- a/tests/func/test_cli.py +++ b/tests/func/test_cli.py @@ -211,6 +211,97 @@ def test_invalid_database_port( } ) + def test_mysql_skip_transfer_data( + self, + cli_runner: CliRunner, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mysql_database: Engine, + ) -> None: + result: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-transfer-data", + ], + ) + assert result.exit_code == 0 + + def test_mysql_skip_create_tables( + self, + cli_runner: CliRunner, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mysql_database: Engine, + ) -> None: + # First we need to create the tables in the MySQL database + result1: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-transfer-data", + ], + ) + assert result1.exit_code == 0 + + result2: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-create-tables", + ], + ) + assert result2.exit_code == 0 + + def test_mysql_skip_create_tables_and_transfer_data( + self, + cli_runner: CliRunner, + sqlite_database: str, + mysql_credentials: MySQLCredentials, + mysql_database: Engine, + ) -> None: + result: Result = cli_runner.invoke( + sqlite3mysql, + [ + "-f", + sqlite_database, + "-d", + mysql_credentials.database, + "-u", + mysql_credentials.user, + "--mysql-password", + mysql_credentials.password, + "--mysql-skip-create-tables", + "--mysql-skip-transfer-data", + ], + ) + assert result.exit_code > 0 + assert ( + "Error: Both -K/--mysql-skip-create-tables and -J/--mysql-skip-transfer-data are set. " + "There is nothing to do. Exiting..." + ) in result.output + @pytest.mark.parametrize( "mysql_integer_type," "mysql_string_type,"