File tree Expand file tree Collapse file tree 2 files changed +271
-1
lines changed
Expand file tree Collapse file tree 2 files changed +271
-1
lines changed Original file line number Diff line number Diff line change @@ -1737,6 +1737,9 @@ def random_shuffle(
17371737 * ,
17381738 seed : Optional [int ] = None ,
17391739 num_blocks : Optional [int ] = None ,
1740+ num_cpus : Optional [float ] = None ,
1741+ num_gpus : Optional [float ] = None ,
1742+ memory : Optional [float ] = None ,
17401743 ** ray_remote_args ,
17411744 ) -> "Dataset" :
17421745 """Randomly shuffle the rows of this :class:`Dataset`.
@@ -1760,6 +1763,11 @@ def random_shuffle(
17601763 Args:
17611764 seed: Fix the random seed to use, otherwise one is chosen
17621765 based on system randomness.
1766+ num_cpus: The number of CPUs to reserve for each parallel shuffle worker.
1767+ num_gpus: The number of GPUs to reserve for each parallel shuffle worker. For
1768+ example, specify `num_gpus=1` to request 1 GPU for each parallel shuffle
1769+ worker.
1770+ memory: The heap memory in bytes to reserve for each parallel shuffle worker.
17631771
17641772 Returns:
17651773 The shuffled :class:`Dataset`.
@@ -1772,6 +1780,12 @@ def random_shuffle(
17721780 "repartition() instead." , # noqa: E501
17731781 )
17741782 plan = self ._plan .copy ()
1783+ ray_remote_args = merge_resources_to_ray_remote_args (
1784+ num_cpus ,
1785+ num_gpus ,
1786+ memory ,
1787+ ray_remote_args ,
1788+ )
17751789 op = RandomShuffle (
17761790 self ._logical_plan .dag ,
17771791 seed = seed ,
You can’t perform that action at this time.
0 commit comments