Merge pull request #9 from jcoleman/master

Refactor string escaping & add with/without timezone option

Author: tardate

Rakefile

@@ -63,9 +63,11 @@ end
 begin
   require 'rcov/rcovtask'
   Rcov::RcovTask.new do |test|
-    test.libs << 'test'
+    test.libs << 'lib'
+    test.libs << 'test/lib'
     test.pattern = 'test/**/*test.rb'
     test.verbose = true
+    test.rcov_opts << "--exclude gems/*"
   end
 rescue LoadError
   task :rcov do

lib/mysql2psql/config.rb

@@ -27,7 +27,7 @@ def reset_configfile(filepath)
       file.close
     end
 
-    def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, force_truncate = false)
+    def self.template(to_filename = nil, include_tables = [], exclude_tables = [], supress_data = false, supress_ddl = false, supress_sequence_update = false, force_truncate = false, use_timezones = false)
       configtext = <<EOS
 mysql:
  hostname: localhost
@@ -83,6 +83,15 @@ def self.template(to_filename = nil, include_tables = [], exclude_tables = [], s
 
 # if supress_ddl is true, only the data will be exported/imported, and not the schema
 supress_ddl: #{supress_ddl}
+EOS
+      end
+      if !supress_sequence_update.nil?
+        configtext += <<EOS
+
+# if supress_sequence_update is true, the sequences for serial (auto-incrementing) columns
+# will not be update to the current maximum value of that column in the database
+# if supress_ddl is not set to true, then this option is implied to be false as well (unless overridden here)
+supress_sequence_update: #{supress_sequence_update}
 EOS
       end
       if !force_truncate.nil?
@@ -92,6 +101,15 @@ def self.template(to_filename = nil, include_tables = [], exclude_tables = [], s
 force_truncate: #{force_truncate}
 EOS
       end
+      if !use_timezones.nil?
+        configtext += <<EOS
+        
+# if use_timezones is true, timestamp/time columns will be created in postgres as "with time zone"
+# rather than "without time zone"
+use_timezones: false
+EOS
+      end
+      
       configtext
     end
 

lib/mysql2psql/converter.rb

@@ -2,7 +2,7 @@ class Mysql2psql
 
   class Converter
     attr_reader :reader, :writer, :options
-    attr_reader :exclude_tables, :only_tables, :supress_data, :supress_ddl, :force_truncate
+    attr_reader :exclude_tables, :only_tables, :supress_data, :supress_ddl, :supress_sequence_update, :force_truncate
 
     def initialize(reader, writer, options)
       @reader = reader
@@ -12,7 +12,9 @@ def initialize(reader, writer, options)
       @only_tables = options.only_tables(nil)
       @supress_data = options.supress_data(false)
       @supress_ddl = options.supress_ddl(false)
+      @supress_sequence_update
       @force_truncate = options.force_truncate(false)
+      @use_timezones = options.use_timezones(false)
     end
 
     def convert
@@ -22,14 +24,17 @@ def convert
         reject {|table| @exclude_tables.include?(table.name)}.
         select {|table| @only_tables ? @only_tables.include?(table.name) : true}
 
+      tables.each do |table|
+        writer.write_sequence_update(table, options)
+      end if !(@supress_sequence_update && @supress_ddl)
 
       tables.each do |table|
-        writer.write_table(table)
+        writer.write_table(table, {:use_timezones => @use_timezones})
       end unless @supress_ddl
 
       _time2 = Time.now
       tables.each do |table|
-        writer.truncate(table) if force_truncate && supress_ddl
+        writer.truncate(table) if force_truncate && !supress_ddl
         writer.write_contents(table, reader)
       end unless @supress_data
 

lib/mysql2psql/postgres_db_writer.rb

@@ -27,47 +27,55 @@ def open
   def close
     @conn.close
   end
-
+  
   def exists?(relname)
     rc = @conn.exec("SELECT COUNT(*) FROM pg_class WHERE relname = '#{relname}'")
     (!rc.nil?) && (rc.to_a.length==1) && (rc.first.count.to_i==1)
   end
 
-  def write_table(table)
+  def write_sequence_update(table, options)
+    serial_key_column = table.columns.detect do |column|
+      column[:auto_increment]
+    end
+    
+    if serial_key_column
+      serial_key = serial_key_column[:name]
+      max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1
+      serial_key_seq = "#{table.name}_#{serial_key}_seq"
+      
+      if !options.supress_ddl
+        if @conn.server_version < 80200
+          @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq)
+        else
+          @conn.exec("DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE")
+        end
+        @conn.exec <<-EOF
+          CREATE SEQUENCE #{serial_key_seq}
+          INCREMENT BY 1
+          NO MAXVALUE
+          NO MINVALUE
+          CACHE 1
+        EOF
+      end
+      
+      if !options.supress_sequence_update
+        puts "Updated sequence #{serial_key_seq} to current value of #{max_value}"
+        @conn.exec sqlfor_set_serial_sequence(table, serial_key_seq, max_value)
+      end
+    end
+  end
+  
+  def write_table(table, options)
     puts "Creating table #{table.name}..."
     primary_keys = []
-    serial_key = nil
-    maxval = nil
 
     columns = table.columns.map do |column|
-      if column[:auto_increment]
-        serial_key = column[:name]
-        maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1
-      end
       if column[:primary_key]
         primary_keys << column[:name]
       end
-      "  " + column_description(column)
+      "  " + column_description(column, options)
     end.join(",\n")
 
-    if serial_key
-      if @conn.server_version < 80200
-        serial_key_seq = "#{table.name}_#{serial_key}_seq"
-        @conn.exec("DROP SEQUENCE #{serial_key_seq} CASCADE") if exists?(serial_key_seq)
-      else
-        @conn.exec("DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE")
-      end
-      @conn.exec <<-EOF
-        CREATE SEQUENCE #{table.name}_#{serial_key}_seq
-        INCREMENT BY 1
-        NO MAXVALUE
-        NO MINVALUE
-        CACHE 1
-      EOF
-    
-      @conn.exec sqlfor_set_serial_sequence(table,serial_key,maxval)
-    end
-    
     if @conn.server_version < 80200
       @conn.exec "DROP TABLE #{PGconn.quote_ident(table.name)} CASCADE;" if exists?(table.name)
     else

lib/mysql2psql/postgres_file_writer.rb

@@ -38,41 +38,53 @@ def truncate(table)
     end
   end
 
-  def write_table(table)
-    primary_keys = []
-    serial_key = nil
-    maxval = nil
-    
-    columns = table.columns.map do |column|
-      if column[:auto_increment]
-        serial_key = column[:name]
-        maxval = column[:maxval].to_i < 1 ? 1 : column[:maxval] + 1
-      end
-      if column[:primary_key]
-        primary_keys << column[:name]
-      end
-      "  " + column_description(column)
-    end.join(",\n")
+  def write_sequence_update(table, options)
+    serial_key_column = table.columns.detect do |column|
+      column[:auto_increment]
+    end
 
-    if serial_key
+    if serial_key_column
+      serial_key = serial_key_column[:name]
+      serial_key_seq = "#{table.name}_#{serial_key}_seq"
+      max_value = serial_key_column[:maxval].to_i < 1 ? 1 : serial_key_column[:maxval] + 1
 
       @f << <<-EOF
 --
--- Name: #{table.name}_#{serial_key}_seq; Type: SEQUENCE; Schema: public
+-- Name: #{serial_key_seq}; Type: SEQUENCE; Schema: public
 --
+EOF
+
+      if !options.supress_ddl
+        @f << <<-EOF
+DROP SEQUENCE IF EXISTS #{serial_key_seq} CASCADE;
  
-DROP SEQUENCE IF EXISTS #{table.name}_#{serial_key}_seq CASCADE;
- 
-CREATE SEQUENCE #{table.name}_#{serial_key}_seq
+CREATE SEQUENCE #{serial_key_seq}
     INCREMENT BY 1
     NO MAXVALUE
     NO MINVALUE
     CACHE 1;
-    
-#{sqlfor_set_serial_sequence(table,serial_key,maxval)} 
- 
-      EOF
+EOF
+      end
+      
+      if !options.supress_sequence_update
+        @f << <<-EOF
+#{sqlfor_set_serial_sequence(table, serial_key_seq, max_value)} 
+EOF
+      end
     end
+  end
+  
+  def write_table(table, options)
+    primary_keys = []
+    serial_key = nil
+    maxval = nil
+    
+    columns = table.columns.map do |column|
+      if column[:primary_key]
+        primary_keys << column[:name]
+      end
+      "  " + column_description(column, options)
+    end.join(",\n")
 
     @f << <<-EOF
 -- Table: #{table.name}

lib/mysql2psql/postgres_writer.rb

@@ -5,15 +5,11 @@
 class Mysql2psql
 
   class PostgresWriter < Writer
-    def column_description(column)
-      "#{PGconn.quote_ident(column[:name])} #{column_type_info(column)}"
+    def column_description(column, options)
+      "#{PGconn.quote_ident(column[:name])} #{column_type_info(column, options)}"
     end
 
-    def column_type(column)
-      column_type_info(column).split(" ").first
-    end
-
-    def column_type(column)
+    def column_type(column, options={})
       if column[:auto_increment]
         'integer'
       else
@@ -29,9 +25,9 @@ def column_type(column)
         when 'decimal'
           "numeric(#{column[:length] || 10}, #{column[:decimals] || 0})"
         when 'datetime', 'timestamp'
-          'timestamp without time zone'
+          "timestamp with#{options[:use_timezones] ? '' : 'out'} time zone"
         when 'time'
-          'time without time zone'
+          "time with#{options[:use_timezones] ? '' : 'out'} time zone"
         when 'tinyblob', 'mediumblob', 'longblob', 'blob', 'varbinary'
           'bytea'
         when 'tinytext', 'mediumtext', 'longtext', 'text'
@@ -97,8 +93,8 @@ def column_default(column)
       end
     end
 
-    def column_type_info(column)
-      type = column_type(column)
+    def column_type_info(column, options)
+      type = column_type(column, options)
       if type
         not_null = !column[:null] || column[:auto_increment] ? ' NOT NULL' : ''
         default = column[:default] || column[:auto_increment] ? " DEFAULT #{column_default(column)}" : ''
@@ -131,7 +127,17 @@ def process_row(table, row)
           if column_type(column) == "bytea"
             row[index] = PGconn.escape_bytea(row[index])
           else
-            row[index] = row[index].gsub(/\\/, '\\\\\\').gsub(/\n/,'\n').gsub(/\t/,'\t').gsub(/\r/,'\r')
+            if row[index] == '\N' || row[index] == '\.'
+              row[index] = '\\' + row[index] # Escape our two PostgreSQL-text-mode-special strings.
+            else
+              # Awesome side-effect producing conditional. Don't do this at home.
+              unless row[index].gsub!(/\0/, '').nil?
+                puts "Removed null bytes from string since PostgreSQL TEXT types don't allow the storage of null bytes."
+              end
+              
+              row[index] = row[index].dump
+              row[index] = row[index].slice(1, row[index].size-2)
+            end
           end
         elsif row[index].nil?
           # Note: '\N' not "\N" is correct here:
@@ -145,11 +151,11 @@ def process_row(table, row)
     def truncate(table)
     end
 
-    def sqlfor_set_serial_sequence(table,serial_key,maxval)
-      "SELECT pg_catalog.setval('#{table.name}_#{serial_key}_seq', #{maxval}, true);"
+    def sqlfor_set_serial_sequence(table, serial_key_seq, max_value)
+      "SELECT pg_catalog.setval('#{serial_key_seq}', #{max_value}, true);"
     end
-    def sqlfor_reset_serial_sequence(table,serial_key,maxval)
-      "SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{maxval}, true);"
+    def sqlfor_reset_serial_sequence(table, serial_key, max_value)
+      "SELECT pg_catalog.setval(pg_get_serial_sequence('#{table.name}', '#{serial_key}'), #{max_value}, true);"
     end
 
   end

test/fixtures/seed_integration_tests.sql

@@ -31,6 +31,7 @@ INSERT INTO numeric_types_basics VALUES
 ( 5,  127, 255, 32767, 65535, 8388607, 16777215, 2147483647, 4294967295, 2147483647, 4294967295, 9223372036854775807, 18446744073709551615, 1, 1, 1, 1, 1, 1);
 
 
+
 DROP TABLE IF EXISTS basic_autoincrement;
 CREATE TABLE basic_autoincrement (
   auto_id INT(11) NOT NULL AUTO_INCREMENT,
@@ -85,3 +86,34 @@ INSERT INTO test_boolean_conversion (test_name, tinyint_1) VALUES ('test-true-no
 CREATE OR REPLACE VIEW test_view AS
 SELECT b.test_name
 FROM test_boolean_conversion b;
+
+DROP TABLE IF EXISTS test_null_conversion;
+CREATE TABLE test_null_conversion (column_a VARCHAR(10));
+INSERT INTO test_null_conversion (column_a) VALUES (NULL);
+
+DROP TABLE IF EXISTS test_datetime_conversion;
+CREATE TABLE test_datetime_conversion (
+	column_a DATETIME,
+	column_b TIMESTAMP,
+	column_c DATETIME DEFAULT '0000-00-00',
+	column_d DATETIME DEFAULT '0000-00-00 00:00',
+	column_e DATETIME DEFAULT '0000-00-00 00:00:00',
+	column_f TIME
+);
+INSERT INTO test_datetime_conversion (column_a, column_f) VALUES ('0000-00-00 00:00', '08:15:30');
+
+DROP TABLE IF EXISTS test_index_conversion;
+CREATE TABLE test_index_conversion (column_a VARCHAR(10));
+CREATE UNIQUE INDEX test_index_conversion ON test_index_conversion (column_a);
+
+DROP TABLE IF EXISTS test_foreign_keys_child;
+DROP TABLE IF EXISTS test_foreign_keys_parent;
+CREATE TABLE test_foreign_keys_parent (id INT NOT NULL, PRIMARY KEY (id)) ENGINE=INNODB;
+CREATE TABLE test_foreign_keys_child (id INT, test_foreign_keys_parent_id INT,
+	INDEX par_ind (test_foreign_keys_parent_id),
+	FOREIGN KEY (test_foreign_keys_parent_id) REFERENCES test_foreign_keys_parent(id) ON DELETE CASCADE
+) ENGINE=INNODB;
+
+DROP TABLE IF EXISTS test_enum;
+CREATE TABLE test_enum (name ENUM('small', 'medium', 'large'));
+INSERT INTO test_enum (name) VALUES ('medium');