Always use the executemany method when inserting rows in DbApiHook as it's way much faster

airflow/providers/common/sql/hooks/sql.py

@@ -519,8 +519,6 @@ def insert_rows(
         target_fields=None,
         commit_every=1000,
         replace=False,
-        *,
-        executemany=False,
         **kwargs,
     ):
         """Insert a collection of tuples into a table.
@@ -534,8 +532,6 @@ def insert_rows(
         :param commit_every: The maximum number of rows to insert in one
             transaction. Set to 0 to insert all rows in one transaction.
         :param replace: Whether to replace instead of insert
-        :param executemany: Insert all rows at once in chunks defined by the commit_every parameter, only
-            works if all rows have same number of column names but leads to better performance
         """
         i = 0
         with closing(self.get_conn()) as conn:
@@ -545,35 +541,19 @@ def insert_rows(
             conn.commit()
 
             with closing(conn.cursor()) as cur:
-                if executemany:
-                    for chunked_rows in chunked(rows, commit_every):
-                        values = list(
-                            map(
-                                lambda row: tuple(map(lambda cell: self._serialize_cell(cell, conn), row)),
-                                chunked_rows,
-                            )
+                for chunked_rows in chunked(rows, commit_every):
+                    values = list(
+                        map(
+                            lambda row: tuple(map(lambda cell: self._serialize_cell(cell, conn), row)),
+                            chunked_rows,
                         )
-                        sql = self._generate_insert_sql(table, values[0], target_fields, replace, **kwargs)
-                        self.log.debug("Generated sql: %s", sql)
-                        cur.fast_executemany = True
-                        cur.executemany(sql, values)
-                        conn.commit()
-                        self.log.info("Loaded %s rows into %s so far", len(chunked_rows), table)
-                else:
-                    for i, row in enumerate(rows, 1):
-                        lst = []
-                        for cell in row:
-                            lst.append(self._serialize_cell(cell, conn))
-                        values = tuple(lst)
-                        sql = self._generate_insert_sql(table, values, target_fields, replace, **kwargs)
-                        self.log.debug("Generated sql: %s", sql)
-                        cur.execute(sql, values)
-                        if commit_every and i % commit_every == 0:
-                            conn.commit()
-                            self.log.info("Loaded %s rows into %s so far", i, table)
-
-            if not executemany:
-                conn.commit()
+                    )
+                    sql = self._generate_insert_sql(table, values[0], target_fields, replace, **kwargs)
+                    self.log.debug("Generated sql: %s", sql)
+                    cur.fast_executemany = True
+                    cur.executemany(sql, values)
+                    conn.commit()
+                    self.log.info("Loaded %s rows into %s so far", len(chunked_rows), table)
         self.log.info("Done loading. Loaded a total of %s rows into %s", i, table)
 
     @staticmethod

tests/providers/common/sql/hooks/test_dbapi.py

@@ -21,6 +21,7 @@
 from unittest import mock
 
 import pytest
+from pyodbc import Cursor
 
 from airflow.hooks.base import BaseHook
 from airflow.models import Connection
@@ -39,7 +40,7 @@ class TestDbApiHook:
     def setup_method(self, **kwargs):
         self.cur = mock.MagicMock(
             rowcount=0,
-            spec=["description", "rowcount", "execute", "executemany", "fetchall", "fetchone", "close"],
+            spec=Cursor,
         )
         self.conn = mock.MagicMock()
         self.conn.cursor.return_value = self.cur
@@ -110,8 +111,7 @@ def test_insert_rows(self):
         assert commit_count == self.conn.commit.call_count
 
         sql = f"INSERT INTO {table}  VALUES (%s)"
-        for row in rows:
-            self.cur.execute.assert_any_call(sql, row)
+        self.cur.executemany.assert_any_call(sql, rows)
 
     def test_insert_rows_replace(self):
         table = "table"
@@ -126,8 +126,7 @@ def test_insert_rows_replace(self):
         assert commit_count == self.conn.commit.call_count
 
         sql = f"REPLACE INTO {table}  VALUES (%s)"
-        for row in rows:
-            self.cur.execute.assert_any_call(sql, row)
+        self.cur.executemany.assert_any_call(sql, rows)
 
     def test_insert_rows_target_fields(self):
         table = "table"
@@ -143,8 +142,7 @@ def test_insert_rows_target_fields(self):
         assert commit_count == self.conn.commit.call_count
 
         sql = f"INSERT INTO {table} ({target_fields[0]}) VALUES (%s)"
-        for row in rows:
-            self.cur.execute.assert_any_call(sql, row)
+        self.cur.executemany.assert_any_call(sql, rows)
 
     def test_insert_rows_commit_every(self):
         table = "table"
@@ -156,32 +154,19 @@ def test_insert_rows_commit_every(self):
         assert self.conn.close.call_count == 1
         assert self.cur.close.call_count == 1
 
-        commit_count = 2 + divmod(len(rows), commit_every)[0]
+        commit_count = len(rows) + 1
         assert commit_count == self.conn.commit.call_count
 
         sql = f"INSERT INTO {table}  VALUES (%s)"
         for row in rows:
-            self.cur.execute.assert_any_call(sql, row)
-
-    def test_insert_rows_executemany(self):
-        table = "table"
-        rows = [("hello",), ("world",)]
-
-        self.db_hook.insert_rows(table, rows, executemany=True)
-
-        assert self.conn.close.call_count == 1
-        assert self.cur.close.call_count == 1
-        assert self.conn.commit.call_count == 2
-
-        sql = f"INSERT INTO {table}  VALUES (%s)"
-        self.cur.executemany.assert_any_call(sql, rows)
+            self.cur.executemany.assert_any_call(sql, [row])
 
     def test_insert_rows_replace_executemany_hana_dialect(self):
         self.setup_method(replace_statement_format="UPSERT {} {} VALUES ({}) WITH PRIMARY KEY")
         table = "table"
         rows = [("hello",), ("world",)]
 
-        self.db_hook.insert_rows(table, rows, replace=True, executemany=True)
+        self.db_hook.insert_rows(table, rows, replace=True)
 
         assert self.conn.close.call_count == 1
         assert self.cur.close.call_count == 1