Centralized runtime control of Edge Worker concurrency in distributed deployments (apache#62896)

Author: dheerajturaga

providers/edge3/src/airflow/providers/edge3/cli/definition.py

@@ -59,6 +59,12 @@
     help="Comma delimited list of queues to add or remove.",
     required=True,
 )
+ARG_CONCURRENCY_REQUIRED = Arg(
+    ("-c", "--concurrency"),
+    type=int,
+    help="The number of worker processes. Must be a positive integer.",
+    required=True,
+)
 ARG_WAIT_MAINT = Arg(
     ("-w", "--wait"),
     default=False,
@@ -229,6 +235,15 @@
         func=lazy_load_command("airflow.providers.edge3.cli.edge_command.shutdown_all_workers"),
         args=(ARG_YES,),
     ),
+    ActionCommand(
+        name="set-worker-concurrency",
+        help="Set the concurrency of a remote edge worker.",
+        func=lazy_load_command("airflow.providers.edge3.cli.edge_command.set_remote_worker_concurrency"),
+        args=(
+            ARG_REQUIRED_EDGE_HOSTNAME,
+            ARG_CONCURRENCY_REQUIRED,
+        ),
+    ),
 ]
 
 

providers/edge3/src/airflow/providers/edge3/cli/edge_command.py

@@ -427,3 +427,27 @@ def remove_worker_queues(args) -> None:
     except TypeError as e:
         logger.error(str(e))
         raise SystemExit
+
+
+@cli_utils.action_cli(check_db=False)
+@providers_configuration_loaded
+def set_remote_worker_concurrency(args) -> None:
+    """Set the concurrency of a remote edge worker."""
+    _check_valid_db_connection()
+    _check_if_registered_edge_host(hostname=args.edge_hostname)
+    from airflow.providers.edge3.models.edge_worker import set_worker_concurrency
+
+    if args.concurrency <= 0:
+        raise SystemExit("Error: Concurrency must be a positive integer.")
+
+    try:
+        set_worker_concurrency(args.edge_hostname, args.concurrency)
+        logger.info(
+            "Concurrency set to %d for Edge Worker host %s by %s.",
+            args.concurrency,
+            args.edge_hostname,
+            getuser(),
+        )
+    except TypeError as e:
+        logger.error(str(e))
+        raise SystemExit

providers/edge3/src/airflow/providers/edge3/cli/worker.py

@@ -401,6 +401,13 @@ async def heartbeat(self, new_maintenance_comments: str | None = None) -> bool:
                 new_maintenance_comments,
             )
             self.queues = worker_info.queues
+            if worker_info.concurrency is not None and worker_info.concurrency != self.concurrency:
+                logger.info(
+                    "Concurrency updated from %d to %d by remote request.",
+                    self.concurrency,
+                    worker_info.concurrency,
+                )
+                self.concurrency = worker_info.concurrency
             if worker_info.state == EdgeWorkerState.MAINTENANCE_REQUEST:
                 logger.info("Maintenance mode requested!")
                 self.maintenance_mode = True

providers/edge3/src/airflow/providers/edge3/migrations/versions/0002_3_2_0_add_concurrency_to_edge_worker.py

@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add concurrency column to edge_worker table.
+
+Revision ID: b3c4d5e6f7a8
+Revises: 9d34dfc2de06
+Create Date: 2026-03-04 00:00:00.000000
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b3c4d5e6f7a8"
+down_revision = "9d34dfc2de06"
+branch_labels = None
+depends_on = None
+edge3_version = "3.2.0"
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    existing_columns = {col["name"] for col in inspector.get_columns("edge_worker")}
+    if "concurrency" not in existing_columns:
+        op.add_column("edge_worker", sa.Column("concurrency", sa.Integer(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("edge_worker", "concurrency")

providers/edge3/src/airflow/providers/edge3/models/db.py

@@ -31,6 +31,7 @@
 
 _REVISION_HEADS_MAP: dict[str, str] = {
     "3.0.0": "9d34dfc2de06",
+    "3.2.0": "b3c4d5e6f7a8",
 }
 
 
@@ -45,6 +46,33 @@ class EdgeDBManager(BaseDBManager):
     supports_table_dropping = True
     revision_heads_map = _REVISION_HEADS_MAP
 
+    def initdb(self):
+        """
+        Initialize the database, handling pre-alembic installations.
+
+        If the edge3 tables already exist but the alembic version table does not
+        (e.g. created via create_all before the migration chain was introduced),
+        stamp to the first revision and run the incremental upgrade so every
+        migration is applied rather than jumping straight to head.
+        """
+        db_exists = self.get_current_revision()
+        if db_exists:
+            self.upgradedb()
+        else:
+            from airflow import settings
+
+            existing_tables = set(inspect(settings.engine).get_table_names())
+            if any(table in existing_tables for table in self.metadata.tables):
+                script = self.get_script_object()
+                base_revision = next(r.revision for r in script.walk_revisions() if r.down_revision is None)
+                config = self.get_alembic_config()
+                from alembic import command
+
+                command.stamp(config, base_revision)
+                self.upgradedb()
+            else:
+                self.create_db_from_orm()
+
     def drop_tables(self, connection):
         """Drop only edge3 tables in reverse dependency order."""
         if not self.supports_table_dropping:

providers/edge3/src/airflow/providers/edge3/models/edge_worker.py

@@ -103,6 +103,7 @@ class EdgeWorkerModel(Base, LoggingMixin):
     jobs_success: Mapped[int] = mapped_column(Integer, default=0)
     jobs_failed: Mapped[int] = mapped_column(Integer, default=0)
     sysinfo: Mapped[str | None] = mapped_column(String(256))
+    concurrency: Mapped[int | None] = mapped_column(Integer, nullable=True)
 
     def __init__(
         self,
@@ -392,3 +393,23 @@ def remove_worker_queues(worker_name: str, queues: list[str], session: Session =
         logger.error(error_message)
         raise TypeError(error_message)
     worker.remove_queues(queues)
+
+
+@provide_session
+def set_worker_concurrency(worker_name: str, concurrency: int, session: Session = NEW_SESSION) -> None:
+    """Set the concurrency of an edge worker."""
+    query = select(EdgeWorkerModel).where(EdgeWorkerModel.worker_name == worker_name)
+    worker: EdgeWorkerModel | None = session.scalar(query)
+    if not worker:
+        raise ValueError(f"Edge Worker {worker_name} not found in list of registered workers")
+    if worker.state in (
+        EdgeWorkerState.OFFLINE,
+        EdgeWorkerState.OFFLINE_MAINTENANCE,
+        EdgeWorkerState.UNKNOWN,
+    ):
+        error_message = (
+            f"Cannot set concurrency for edge worker {worker_name} as it is in {worker.state} state!"
+        )
+        logger.error(error_message)
+        raise TypeError(error_message)
+    worker.concurrency = concurrency

providers/edge3/src/airflow/providers/edge3/worker_api/datamodels.py

@@ -200,3 +200,10 @@ class WorkerSetStateReturn(BaseModel):
         str | None,
         Field(description="Comments about the maintenance state of the worker."),
     ] = None
+    concurrency: Annotated[
+        int | None,
+        Field(
+            description="Desired concurrency for the worker set by an administrator. "
+            "None means no remote override; the worker uses its startup value.",
+        ),
+    ] = None

providers/edge3/src/airflow/providers/edge3/worker_api/routes/worker.py

@@ -238,7 +238,10 @@ def set_state(
     )
     _assert_version(body.sysinfo)  # Exception only after worker state is in the DB
     return WorkerSetStateReturn(
-        state=worker.state, queues=worker.queues, maintenance_comments=worker.maintenance_comment
+        state=worker.state,
+        queues=worker.queues,
+        maintenance_comments=worker.maintenance_comment,
+        concurrency=worker.concurrency,
     )
 
 

providers/edge3/src/airflow/providers/edge3/worker_api/v2-edge-generated.yaml

@@ -1413,6 +1413,13 @@ components:
           - type: 'null'
           title: Maintenance Comments
           description: Comments about the maintenance state of the worker.
+        concurrency:
+          anyOf:
+          - type: integer
+          - type: 'null'
+          title: Concurrency
+          description: Desired concurrency for the worker set by an administrator.
+            None means no remote override; the worker uses its startup value.
       type: object
       required:
       - state

providers/edge3/tests/unit/edge3/cli/test_definition.py

@@ -53,8 +53,8 @@ def test_edge_cli_commands_count(self):
         assert len(commands) == 1
 
     def test_edge_commands_count(self):
-        """Test that EDGE_COMMANDS contains all 13 subcommands."""
-        assert len(EDGE_COMMANDS) == 13
+        """Test that EDGE_COMMANDS contains all 14 subcommands."""
+        assert len(EDGE_COMMANDS) == 14
 
     @pytest.mark.parametrize(
         "command",
@@ -234,3 +234,17 @@ def test_shutdown_all_workers_args(self):
         params = ["edge", "shutdown-all-workers", "--yes"]
         args = self.arg_parser.parse_args(params)
         assert args.yes is True
+
+    def test_set_worker_concurrency_args(self):
+        """Test set-worker-concurrency command with required arguments."""
+        params = [
+            "edge",
+            "set-worker-concurrency",
+            "--edge-hostname",
+            "remote-worker-1",
+            "--concurrency",
+            "16",
+        ]
+        args = self.arg_parser.parse_args(params)
+        assert args.edge_hostname == "remote-worker-1"
+        assert args.concurrency == 16