Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions jax/experimental/pallas/ops/gpu/attention_mgpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def _attention_forward(q, k, v, config: TuningConfig, save_residuals: bool = Fal
kv_shape = (batch_size, kv_seq_len, num_kv_heads, head_dim)
if k.shape != kv_shape:
raise ValueError(f"Expected {k.shape=} to be {kv_shape} (inferred from q)")
if k.shape != kv_shape:
if v.shape != kv_shape:
raise ValueError(f"Expected {v.shape=} to be {kv_shape} (inferred from q)")
if (dtype := q.dtype) != k.dtype or dtype != v.dtype:
raise ValueError(f"q, k, and v should all have the same dtype, got: {q.dtype}, {k.dtype}, {v.dtype}")
Expand Down Expand Up @@ -661,7 +661,7 @@ def attention_with_pipeline_emitter(q, k, v, config: TuningConfig, save_residual
kv_shape = (batch_size, kv_seq_len, num_kv_heads, head_dim)
if k.shape != kv_shape:
raise ValueError(f"Expected {k.shape=} to be {kv_shape} (inferred from q)")
if k.shape != kv_shape:
if v.shape != kv_shape:
raise ValueError(f"Expected {v.shape=} to be {kv_shape} (inferred from q)")
if (dtype := q.dtype) != k.dtype or dtype != v.dtype:
raise ValueError(f"q, k, and v should all have the same dtype, got: {q.dtype}, {k.dtype}, {v.dtype}")
Expand Down
14 changes: 8 additions & 6 deletions tests/pallas/mgpu_attention_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,14 @@ def f(q, k, v):
def f_ref(q, k, v):
return attention_mgpu.attention_reference(q, k, v).sum()

# TODO(pobudzey): Replace with `jtu.check_grads` when it's fixed.
dq_ref, dk_ref, dv_ref = jax.grad(f_ref, argnums=(0, 1, 2))(q, k, v)
dq_ref_host = np.array(dq_ref)
dk_ref_host = np.array(dk_ref)
dv_ref_host = np.array(dv_ref)
del dq_ref, dk_ref, dv_ref
try:
# TODO(pobudzey): Replace with `jtu.check_grads` when it's fixed.
dq_ref, dk_ref, dv_ref = jax.grad(f_ref, argnums=(0, 1, 2))(q, k, v)
dq_ref_host = np.array(dq_ref)
dk_ref_host = np.array(dk_ref)
dv_ref_host = np.array(dv_ref)
del dq_ref, dk_ref, dv_ref


dq, dk, dv = jax.grad(f, argnums=(0, 1, 2))(q, k, v)

Expand All @@ -179,6 +180,7 @@ def f_ref(q, k, v):
except ValueError as e:
if "exceeds available shared memory" in e.args[0]:
self.skipTest("Not enough SMEM for this configuration.")
raise

if __name__ == "__main__":
absltest.main(testLoader=jtu.JaxTestLoader())
Loading