From 1c26cfa25cb9ca7b378fdd491a292521e71c2a36 Mon Sep 17 00:00:00 2001 From: Andrew Eze Date: Sun, 15 Nov 2026 14:16:32 +0100 Subject: [PATCH 1/2] FIX : backward test eating unrelated value errors --- tests/pallas/mgpu_attention_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pallas/mgpu_attention_test.py b/tests/pallas/mgpu_attention_test.py index ec73227b01bd..990522e71362 100644 --- a/tests/pallas/mgpu_attention_test.py +++ b/tests/pallas/mgpu_attention_test.py @@ -179,6 +179,6 @@ def f_ref(q, k, v): except ValueError as e: if "exceeds available shared memory" in e.args[0]: self.skipTest("Not enough SMEM for this configuration.") - + raise if __name__ == "__main__": absltest.main(testLoader=jtu.JaxTestLoader()) From c35ef696d1f7f62105b87652950884f568a8f16f Mon Sep 17 00:00:00 2001 From: Andrew Eze Date: Mon, 16 Nov 2026 18:47:39 +0100 Subject: [PATCH 2/2] Fix multi-GPU attention shape validation --- jax/experimental/pallas/ops/gpu/attention_mgpu.py | 4 ++-- tests/pallas/mgpu_attention_test.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/jax/experimental/pallas/ops/gpu/attention_mgpu.py b/jax/experimental/pallas/ops/gpu/attention_mgpu.py index 55ba62c20f71..247c81cdb5ac 100644 --- a/jax/experimental/pallas/ops/gpu/attention_mgpu.py +++ b/jax/experimental/pallas/ops/gpu/attention_mgpu.py @@ -85,7 +85,7 @@ def _attention_forward(q, k, v, config: TuningConfig, save_residuals: bool = Fal kv_shape = (batch_size, kv_seq_len, num_kv_heads, head_dim) if k.shape != kv_shape: raise ValueError(f"Expected {k.shape=} to be {kv_shape} (inferred from q)") - if k.shape != kv_shape: + if v.shape != kv_shape: raise ValueError(f"Expected {v.shape=} to be {kv_shape} (inferred from q)") if (dtype := q.dtype) != k.dtype or dtype != v.dtype: raise ValueError(f"q, k, and v should all have the same dtype, got: {q.dtype}, {k.dtype}, {v.dtype}") @@ -661,7 +661,7 @@ def attention_with_pipeline_emitter(q, k, v, config: TuningConfig, save_residual kv_shape = (batch_size, kv_seq_len, num_kv_heads, head_dim) if k.shape != kv_shape: raise ValueError(f"Expected {k.shape=} to be {kv_shape} (inferred from q)") - if k.shape != kv_shape: + if v.shape != kv_shape: raise ValueError(f"Expected {v.shape=} to be {kv_shape} (inferred from q)") if (dtype := q.dtype) != k.dtype or dtype != v.dtype: raise ValueError(f"q, k, and v should all have the same dtype, got: {q.dtype}, {k.dtype}, {v.dtype}") diff --git a/tests/pallas/mgpu_attention_test.py b/tests/pallas/mgpu_attention_test.py index 990522e71362..2673ac295509 100644 --- a/tests/pallas/mgpu_attention_test.py +++ b/tests/pallas/mgpu_attention_test.py @@ -162,13 +162,14 @@ def f(q, k, v): def f_ref(q, k, v): return attention_mgpu.attention_reference(q, k, v).sum() + # TODO(pobudzey): Replace with `jtu.check_grads` when it's fixed. + dq_ref, dk_ref, dv_ref = jax.grad(f_ref, argnums=(0, 1, 2))(q, k, v) + dq_ref_host = np.array(dq_ref) + dk_ref_host = np.array(dk_ref) + dv_ref_host = np.array(dv_ref) + del dq_ref, dk_ref, dv_ref try: - # TODO(pobudzey): Replace with `jtu.check_grads` when it's fixed. - dq_ref, dk_ref, dv_ref = jax.grad(f_ref, argnums=(0, 1, 2))(q, k, v) - dq_ref_host = np.array(dq_ref) - dk_ref_host = np.array(dk_ref) - dv_ref_host = np.array(dv_ref) - del dq_ref, dk_ref, dv_ref + dq, dk, dv = jax.grad(f, argnums=(0, 1, 2))(q, k, v) @@ -180,5 +181,6 @@ def f_ref(q, k, v): if "exceeds available shared memory" in e.args[0]: self.skipTest("Not enough SMEM for this configuration.") raise + if __name__ == "__main__": absltest.main(testLoader=jtu.JaxTestLoader())