Repository navigation
Description
Description
pltpu.sync_copy(source, shared.at[positions]) fails to compile when source is storage.at[1] from a (2, W) SC TileSPMEM Ref for W = 256, 512, and 1024. The same operation compiles when source is the contiguous slice storage.at[pl.ds(W, W)] from a (2 * W,) Ref.
Save the following as repro_scatter_row.py. It compares both variants offline and does not require a TPU:
"""Compile indirect scatter from a row or a contiguous 1-D slice."""
import sys
import jax
import jax.numpy as jnp
import numpy as np
from jax.experimental import pallas as pl
from jax.experimental import topologies
from jax.experimental.pallas import tpu as pltpu
from jax.experimental.pallas import tpu_sc as plsc
from jax.sharding import Mesh, NamedSharding, PartitionSpec as P
topology = topologies.get_topology_desc(platform='tpu', topology_name='tpu7x:2x2x1')
devices = Mesh(np.asarray(topology.devices[:1]), ('device',))
def compile_scatter(width: int, row: bool) -> None:
with jax.sharding.use_abstract_mesh(devices.abstract_mesh):
@pl.kernel(
out_type=jax.ShapeDtypeStruct((16 * width,), jnp.int32),
mesh=plsc.VectorSubcoreMesh(core_axis_name='core', subcore_axis_name='subcore', num_cores=1, num_subcores=16),
scratch_types=(
pltpu.VMEM((2, width) if row else (2 * width,), jnp.int32),
pltpu.VMEM((width,), jnp.int32),
pltpu.VMEM((width,), jnp.int32),
pltpu.VMEM_SHARED((16 * width,), jnp.int32),
),
compiler_params=pltpu.CompilerParams(
needs_layout_passes=False,
disable_bounds_checks=True,
disable_semaphore_checks=True,
skip_device_barrier=True,
),
)
def kernel(out_hbm: jax.Ref, storage: jax.Ref, positions: jax.Ref, received: jax.Ref, shared: jax.Ref) -> None:
t = jax.lax.axis_index('subcore')
source = storage.at[1] if row else storage.at[pl.ds(width, width)]
@pl.loop(0, width // 16)
def _(g: jax.Array) -> None:
w = pl.ds(pl.multiple_of(g * 16, 16), 16)
j = g * 16 + jnp.arange(16, dtype=jnp.int32)
source[w] = t * width + j
positions[w] = t * width + width - 1 - j
pltpu.sync_copy(source, shared.at[positions])
plsc.subcore_barrier()
pltpu.sync_copy(shared.at[pl.ds(t * width, width)], received)
pltpu.sync_copy(received, out_hbm.at[pl.ds(t * width, width)])
jax.jit(kernel, out_shardings=NamedSharding(devices, P())).trace().lower(lowering_platforms=('tpu',)).compile()
if __name__ == '__main__':
width = int(sys.argv[1])
compile_scatter(width, row=False)
print(f'W = {width}, 1-D slice: compiled', flush=True)
compile_scatter(width, row=True)
print(f'W = {width}, row of (2, W): compiled', flush=True)Run:
for width in 16 64 128 256 512 1024; do
JAX_PLATFORMS=cpu JAX_ENABLE_COMPILATION_CACHE=false \
python repro_scatter_row.py "$width"
doneExpected behavior
The compilation is successful for every tested width.
Actual behavior
| W | Contiguous slice of (2 * W,) |
Row of (2, W) |
|---|---|---|
| 16 | Compiles | Compiles |
| 64 | Compiles | Compiles |
| 128 | Compiles | Compiles |
| 256 | Compiles | Fails in ExpandTiledMemRefs |
| 512 | Compiles | Fails in ExpandTiledMemRefs |
| 1024 | Compiles | Fails in ExpandTiledMemRefs |
For W = 128:
W = 128, 1-D slice: compiled
W = 128, row of (2, W): compiled
For W = 256, the contiguous source compiles and the row source fails as follows. Startup diagnostics are omitted and local file paths are abbreviated.
W = 256, 1-D slice: compiled
Traceback (most recent call last):
File "repro_scatter_row.py", line 57, in <module>
compile_scatter(width, row=True)
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
File "repro_scatter_row.py", line 51, in compile_scatter
jax.jit(kernel, out_shardings=NamedSharding(devices, P())).trace().lower(lowering_platforms=('tpu',)).compile()
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
File ".../site-packages/jax/_src/stages.py", line 721, in compile
return Compiled(self._lowering.compile(**kw), self._lowering.const_args,
~~~~~~~~~~~~~~~~~~~~~~^^^^^^
File ".../site-packages/jax/_src/interpreters/pxla.py", line 1237, in compile
executable = UnloadedMeshExecutable.from_hlo(
self._name, self._hlo, **self.compile_args,
compiler_options_kvs=compiler_options_kvs,
device_list=compilation_device_list,
num_const_args=len(self.const_args))
File ".../site-packages/jax/_src/interpreters/pxla.py", line 1736, in from_hlo
xla_executable = _cached_compilation(
hlo, name, tuple_args, allow_prop_to_inputs, allow_prop_to_outputs,
tuple(host_callbacks), backend, device_list, compiler_options_kvs,
pgle_profiler)
File ".../site-packages/jax/_src/interpreters/pxla.py", line 1515, in _cached_compilation
xla_executable = compiler.compile_or_get_cached(
backend, computation, dev, compile_options, host_callbacks,
da, pgle_profiler)
File ".../site-packages/jax/_src/compiler.py", line 471, in compile_or_get_cached
return backend_compile_and_load(
backend, computation, executable_devices, compile_options,
host_callbacks)
File ".../site-packages/jax/_src/profiler.py", line 574, in wrapper
return func(*args, **kwargs)
File ".../site-packages/jax/_src/compiler.py", line 412, in backend_compile_and_load
raise e
File ".../site-packages/jax/_src/compiler.py", line 386, in backend_compile_and_load
return backend.compile( # pyrefly: ignore[bad-return]
~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
module,
^^^^^^^
executable_devices=executable_devices,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
compile_options=options,
^^^^^^^^^^^^^^^^^^^^^^^^
)
^
jax.errors.JaxRuntimeError: INTERNAL: Failed to run MLO pass pipeline.wrapper.6:58:1: error: 'sc_tpu.enqueue_indirect_transfer' op Expected source and target to have 2D tiles, but got target (scatter operand) tile (128) and source (scatter updates) tile (128).
wrapper.6:58:1: note: see current operation: "sc_tpu.enqueue_indirect_transfer"(%149, %37, %33, %143) <{add = false, enable_trace = false, operandSegmentSizes = array<i32: 1, 1, 1, 1, 0>}> : (memref<256xi32, #xir.tiled<(128),[2]>, #sc_tpu.memory_space<tile_spmem>>, memref<4096xi32, #xir.tiled<(128),[1]>, #sc_tpu.memory_space<spmem>>, memref<256xi32, #xir.tiled<(128),[1]>, #sc_tpu.memory_space<tile_spmem>>, memref<i32, #xir.tiled<,[]>, #sc_tpu.memory_space<sflag_tile>>) -> ()
wrapper.6:58:1: error: failed to legalize operation 'sc_tpu.enqueue_indirect_transfer': "sc_tpu.enqueue_indirect_transfer"(%149, %37, %33, %143) <{add = false, enable_trace = false, operandSegmentSizes = array<i32: 1, 1, 1, 1, 0>}> : (memref<256xi32, #xir.tiled<(128),[2]>, #sc_tpu.memory_space<tile_spmem>>, memref<4096xi32, #xir.tiled<(128),[1]>, #sc_tpu.memory_space<spmem>>, memref<256xi32, #xir.tiled<(128),[1]>, #sc_tpu.memory_space<tile_spmem>>, memref<i32, #xir.tiled<,[]>, #sc_tpu.memory_space<sflag_tile>>) -> ()
wrapper.6:58:1: note: see current operation: "sc_tpu.enqueue_indirect_transfer"(%149, %37, %33, %143) <{add = false, enable_trace = false, operandSegmentSizes = array<i32: 1, 1, 1, 1, 0>}> : (memref<256xi32, #xir.tiled<(128),[2]>, #sc_tpu.memory_space<tile_spmem>>, memref<4096xi32, #xir.tiled<(128),[1]>, #sc_tpu.memory_space<spmem>>, memref<256xi32, #xir.tiled<(128),[1]>, #sc_tpu.memory_space<tile_spmem>>, memref<i32, #xir.tiled<,[]>, #sc_tpu.memory_space<sflag_tile>>) -> ()
main.1:1:1: error: NOTE: Failed in pass ExpandTiledMemRefsNotes
The failing operation shows the source as memref<256xi32, #xir.tiled<(128),[2]>, #sc_tpu.memory_space<tile_spmem>>. Selecting a row leaves a one-dimensional view whose tiles contain 128 elements and whose tile stride is 2. Successive tiles of that row are therefore separated by a tile belonging to the other row. The destination and index Ref have tile stride 1.
The tested widths up to 128 occupy a single source tile, whereas W = 256, 512, and 1024 require multiple source tiles. All six contiguous one-dimensional source variants compile. Together with the diagnostic that expects two-dimensional tiles, this suggests a missing case for a strided one-dimensional source in indirect-transfer lowering.
System info (python version, jaxlib version, accelerator, etc.)
jax: 0.12.1.dev20261111+87648ea819
jaxlib: 0.12.1.dev20261111
libtpu: 0.0.53.dev20261111+nightly
numpy: 2.5.4
python: 3.14.8 free-threading build (main, Oct 9 2026, 15:23:34) [Clang 22.1.3 ]
device info: cpu-1, 1 local devices"
process_count: 1
platform: uname_result(system='Linux', node='<redacted>', release='7.2.14-arch1-1', version='#1 SMP PREEMPT_DYNAMIC Fri, 30 Oct 2026 14:32:40 +0000', machine='x86_64')
Activity