Skip to content

Commit 4246907

Browse files
jskswamyandreyvelich
authored and
mahdikhashan
committed
feat(runtimes): add support for launcher resource allocation in MPI jobs (kubeflow#2653)
* feat(runtime): add support for launcher resource allocation in MPI jobs Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> * Add unit tests Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> * Set numProcPerNode for MPI plugin Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> * Move util func to runtime package Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> * Fix torchtune plugin Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> * Inline if for GPU check Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> * Assign container resources once Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> * Add todo for test wrappers Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> --------- Signed-off-by: Andrey Velichkevich <andrey.velichkevich@gmail.com> Co-authored-by: Andrey Velichkevich <andrey.velichkevich@gmail.com>
1 parent ca37406 commit 4246907

11 files changed

Lines changed: 2027 additions & 1056 deletions

File tree

‎examples/deepspeed/text-summarization/T5-Fine-Tuning.ipynb‎

Lines changed: 475 additions & 411 deletions
Large diffs are not rendered by default.

‎examples/mlx/image-classification/mnist.ipynb‎

Lines changed: 346 additions & 398 deletions
Large diffs are not rendered by default.

‎examples/mlx/language-modeling/fine-tune-llama.ipynb‎

Lines changed: 143 additions & 182 deletions
Large diffs are not rendered by default.

‎pkg/runtime/framework/core/framework_test.go‎

Lines changed: 834 additions & 2 deletions
Large diffs are not rendered by default.

‎pkg/runtime/framework/plugins/jobset/builder.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,21 +101,26 @@ func (b *Builder) Initializer(trainJob *trainer.TrainJob) *Builder {
101101
return b
102102
}
103103

104+
// isRunLauncherAsNode returns true if runLauncherAsNode is set to true in the MPI policy.
105+
func (b *Builder) isRunLauncherAsNode(info *runtime.Info) bool {
106+
return info.RuntimePolicy.MLPolicySource != nil &&
107+
info.RuntimePolicy.MLPolicySource.MPI != nil &&
108+
info.RuntimePolicy.MLPolicySource.MPI.RunLauncherAsNode != nil &&
109+
*info.RuntimePolicy.MLPolicySource.MPI.RunLauncherAsNode
110+
}
111+
104112
// Trainer updates JobSet values for the trainer Job.
105113
func (b *Builder) Trainer(info *runtime.Info, trainJob *trainer.TrainJob) *Builder {
106114
for i, rJob := range b.Spec.ReplicatedJobs {
115+
ancestor := ""
107116
jobMetadata := rJob.Template.ObjectMetaApplyConfiguration
108-
if jobMetadata == nil || jobMetadata.Labels == nil {
109-
continue
117+
if jobMetadata != nil && jobMetadata.Labels != nil {
118+
ancestor = jobMetadata.Labels[constants.LabelTrainJobAncestor]
110119
}
111-
if ancestor, ok := jobMetadata.Labels[constants.LabelTrainJobAncestor]; ok && ancestor == constants.AncestorTrainer {
120+
if ancestor == constants.AncestorTrainer {
112121
// TODO: Support multiple replicas ('.template.spec.replicatedJobs[*].replicas') for replicated Jobs.
113122
// REF: https://github.com/kubeflow/trainer/issues/2318
114123
b.Spec.ReplicatedJobs[i].Replicas = ptr.To[int32](1)
115-
// Update the Parallelism and Completions values for the Trainer Job.
116-
b.Spec.ReplicatedJobs[i].Template.Spec.Parallelism = info.FindPodSetByAncestor(constants.AncestorTrainer).Count
117-
b.Spec.ReplicatedJobs[i].Template.Spec.Completions = info.FindPodSetByAncestor(constants.AncestorTrainer).Count
118-
119124
// Update values for the Trainer container.
120125
for j, container := range rJob.Template.Spec.Template.Spec.Containers {
121126
if *container.Name == constants.Node {
@@ -130,6 +135,16 @@ func (b *Builder) Trainer(info *runtime.Info, trainJob *trainer.TrainJob) *Build
130135
if args := jobTrainer.Args; args != nil {
131136
b.Spec.ReplicatedJobs[i].Template.Spec.Template.Spec.Containers[j].Args = args
132137
}
138+
}
139+
}
140+
}
141+
}
142+
if ancestor == constants.AncestorTrainer || b.isRunLauncherAsNode(info) && *rJob.Name == constants.Node {
143+
// TODO (andreyvelich): For MPI we should apply container resources to the Node ReplicatedJob also.
144+
// Eventually, we should find better way to propagate resources from TrainJob to JobSet.
145+
for j, container := range rJob.Template.Spec.Template.Spec.Containers {
146+
if *container.Name == constants.Node {
147+
if jobTrainer := trainJob.Spec.Trainer; jobTrainer != nil {
133148
if resourcesPerNode := jobTrainer.ResourcesPerNode; resourcesPerNode != nil &&
134149
(resourcesPerNode.Limits != nil || resourcesPerNode.Requests != nil) {
135150
requirements := corev1ac.ResourceRequirements()

‎pkg/runtime/framework/plugins/mpi/mpi.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,6 @@ func (m *MPI) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) er
113113
if trainJob.Spec.Trainer != nil && trainJob.Spec.Trainer.NumNodes != nil {
114114
if node := info.FindPodSetByName(constants.Node); node != nil && node.Count != nil {
115115
if ptr.Deref(info.RuntimePolicy.MLPolicySource.MPI.RunLauncherAsNode, false) {
116-
// TODO: We should implement more strong validations for the MPIRuntime with runLauncherAsNode.
117-
// REF: https://github.com/kubeflow/trainer/issues/2550
118116
// When runLauncherAsNode is enabled, 1 nodes should be allocated to launcher.
119117
*node.Count = max(*trainJob.Spec.Trainer.NumNodes-1, 1)
120118
} else {
@@ -125,6 +123,15 @@ func (m *MPI) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) er
125123

126124
if trainJob.Spec.Trainer != nil && trainJob.Spec.Trainer.NumProcPerNode != nil {
127125
info.RuntimePolicy.MLPolicySource.MPI.NumProcPerNode = ptr.To(int32(trainJob.Spec.Trainer.NumProcPerNode.IntValue()))
126+
// If numProcPerNode is set to 1 in runtime, we make it equal to number of GPUs.
127+
} else if *info.RuntimePolicy.MLPolicySource.MPI.NumProcPerNode == 1 {
128+
resourcesPerNode := ptr.Deref(runtime.ExtractResourcePerNodeFromRuntime(info), corev1.ResourceRequirements{})
129+
if jobTrainer := trainJob.Spec.Trainer; jobTrainer != nil && jobTrainer.ResourcesPerNode != nil {
130+
resourcesPerNode = ptr.Deref(jobTrainer.ResourcesPerNode, corev1.ResourceRequirements{})
131+
}
132+
if gpuQ := runtime.GetNumGPUPerNode(&resourcesPerNode); gpuQ > 1 {
133+
info.RuntimePolicy.MLPolicySource.MPI.NumProcPerNode = ptr.To(int32(gpuQ))
134+
}
128135
}
129136

130137
// Add Secret and ConfigMap volumes to the Info object

‎pkg/runtime/framework/plugins/mpi/mpi_test.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
gocmp "github.com/google/go-cmp/cmp"
2727
"github.com/google/go-cmp/cmp/cmpopts"
2828
corev1 "k8s.io/api/core/v1"
29+
"k8s.io/apimachinery/pkg/api/resource"
2930
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3031
apiruntime "k8s.io/apimachinery/pkg/runtime"
3132
"k8s.io/apimachinery/pkg/util/intstr"
@@ -350,6 +351,143 @@ trainJob-node-1-1.trainJob slots=1
350351
utiltesting.MakeConfigMapWrapper(fmt.Sprintf("trainJob%s", constants.MPIHostfileConfigMapSuffix), metav1.NamespaceDefault).
351352
WithData(map[string]string{
352353
constants.MPIHostfileName: `trainJob-node-1-0.trainJob slots=2
354+
`,
355+
}).
356+
ControllerReference(trainer.SchemeGroupVersion.WithKind(trainer.TrainJobKind), "trainJob", "trainJob").
357+
Obj(),
358+
},
359+
},
360+
"numProcPerNode is set to number of GPUs in TrainJob": {
361+
info: &runtime.Info{
362+
Labels: make(map[string]string),
363+
Annotations: make(map[string]string),
364+
TemplateSpec: runtime.TemplateSpec{
365+
PodSets: []runtime.PodSet{
366+
{
367+
Name: constants.Launcher,
368+
Count: ptr.To[int32](1),
369+
Endpoints: func(yield func(string) bool) {
370+
yield("trainJob-launcher-0-0.trainJob")
371+
},
372+
},
373+
{
374+
Name: constants.Node,
375+
Ancestor: ptr.To(constants.AncestorTrainer),
376+
Count: ptr.To[int32](1),
377+
Endpoints: func(yield func(string) bool) {
378+
yield("trainJob-node-1-0.trainJob")
379+
},
380+
},
381+
},
382+
},
383+
RuntimePolicy: runtime.RuntimePolicy{
384+
MLPolicySource: utiltesting.MakeMLPolicySourceWrapper().
385+
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, ptr.To("/root/.ssh"), nil).
386+
Obj(),
387+
},
388+
Scheduler: &runtime.Scheduler{
389+
PodLabels: make(map[string]string),
390+
},
391+
},
392+
trainJob: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "trainJob").
393+
UID("trainJob").
394+
Trainer(
395+
utiltesting.MakeTrainJobTrainerWrapper().
396+
NumNodes(1).
397+
Container("test:trainjob", []string{"trainjob"}, []string{"trainjob"}, corev1.ResourceList{
398+
"custom.com/gpu": resource.MustParse("5"),
399+
}).
400+
Obj()).
401+
Obj(),
402+
wantInfo: &runtime.Info{
403+
Labels: make(map[string]string),
404+
Annotations: make(map[string]string),
405+
TemplateSpec: runtime.TemplateSpec{
406+
PodSets: []runtime.PodSet{
407+
{
408+
Name: constants.Launcher,
409+
Count: ptr.To[int32](1),
410+
Volumes: []corev1ac.VolumeApplyConfiguration{
411+
*corev1ac.Volume().
412+
WithName(constants.MPISSHAuthVolumeName).
413+
WithSecret(corev1ac.SecretVolumeSource().
414+
WithSecretName(fmt.Sprintf("trainJob%s", constants.MPISSHAuthSecretSuffix)).
415+
WithItems(
416+
corev1ac.KeyToPath().
417+
WithKey(corev1.SSHAuthPrivateKey).
418+
WithPath(constants.MPISSHPrivateKeyFile),
419+
corev1ac.KeyToPath().
420+
WithKey(constants.MPISSHPublicKey).
421+
WithPath(constants.MPISSHPublicKeyFile),
422+
corev1ac.KeyToPath().
423+
WithKey(constants.MPISSHPublicKey).
424+
WithPath(constants.MPISSHAuthorizedKeys),
425+
),
426+
),
427+
*corev1ac.Volume().
428+
WithName(constants.MPIHostfileVolumeName).
429+
WithConfigMap(corev1ac.ConfigMapVolumeSource().
430+
WithName(fmt.Sprintf("trainJob%s", constants.MPIHostfileConfigMapSuffix)).
431+
WithItems(
432+
corev1ac.KeyToPath().
433+
WithKey(constants.MPIHostfileName).
434+
WithPath(constants.MPIHostfileName).
435+
WithMode(0444),
436+
),
437+
),
438+
},
439+
Endpoints: func(yield func(string) bool) {
440+
yield("trainJob-launcher-0-0.trainJob")
441+
},
442+
},
443+
{
444+
Name: constants.Node,
445+
Ancestor: ptr.To(constants.AncestorTrainer),
446+
Count: ptr.To[int32](1),
447+
Volumes: []corev1ac.VolumeApplyConfiguration{
448+
*corev1ac.Volume().
449+
WithName(constants.MPISSHAuthVolumeName).
450+
WithSecret(corev1ac.SecretVolumeSource().
451+
WithSecretName(fmt.Sprintf("trainJob%s", constants.MPISSHAuthSecretSuffix)).
452+
WithItems(
453+
corev1ac.KeyToPath().
454+
WithKey(corev1.SSHAuthPrivateKey).
455+
WithPath(constants.MPISSHPrivateKeyFile),
456+
corev1ac.KeyToPath().
457+
WithKey(constants.MPISSHPublicKey).
458+
WithPath(constants.MPISSHPublicKeyFile),
459+
corev1ac.KeyToPath().
460+
WithKey(constants.MPISSHPublicKey).
461+
WithPath(constants.MPISSHAuthorizedKeys),
462+
),
463+
),
464+
},
465+
Endpoints: func(yield func(string) bool) {
466+
yield("trainJob-node-1-0.trainJob")
467+
},
468+
},
469+
},
470+
},
471+
RuntimePolicy: runtime.RuntimePolicy{
472+
MLPolicySource: utiltesting.MakeMLPolicySourceWrapper().
473+
MPIPolicy(ptr.To[int32](5), trainer.MPIImplementationOpenMPI, ptr.To("/root/.ssh"), nil).
474+
Obj(),
475+
},
476+
Scheduler: &runtime.Scheduler{PodLabels: make(map[string]string)},
477+
},
478+
wantObjs: []apiruntime.Object{
479+
utiltesting.MakeSecretWrapper(fmt.Sprintf("trainJob%s", constants.MPISSHAuthSecretSuffix), metav1.NamespaceDefault).
480+
WithImmutable(true).
481+
WithType(corev1.SecretTypeSSHAuth).
482+
WithData(map[string][]byte{
483+
constants.MPISSHPublicKey: []byte("EXIST"),
484+
corev1.SSHAuthPrivateKey: []byte("EXIST"),
485+
}).
486+
ControllerReference(trainer.SchemeGroupVersion.WithKind(trainer.TrainJobKind), "trainJob", "trainJob").
487+
Obj(),
488+
utiltesting.MakeConfigMapWrapper(fmt.Sprintf("trainJob%s", constants.MPIHostfileConfigMapSuffix), metav1.NamespaceDefault).
489+
WithData(map[string]string{
490+
constants.MPIHostfileName: `trainJob-node-1-0.trainJob slots=5
353491
`,
354492
}).
355493
ControllerReference(trainer.SchemeGroupVersion.WithKind(trainer.TrainJobKind), "trainJob", "trainJob").

‎pkg/runtime/framework/plugins/torch/torch.go‎

Lines changed: 2 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"context"
2121
"fmt"
2222
"slices"
23-
"strings"
2423

2524
corev1 "k8s.io/api/core/v1"
2625
"k8s.io/apimachinery/pkg/util/intstr"
@@ -30,7 +29,6 @@ import (
3029
"k8s.io/utils/ptr"
3130
"sigs.k8s.io/controller-runtime/pkg/client"
3231
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
33-
jobsetv1alpha2ac "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2"
3432

3533
trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1"
3634
"github.com/kubeflow/trainer/v2/pkg/apply"
@@ -113,11 +111,11 @@ func (t *Torch) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob)
113111
}
114112

115113
// Determine numProcPerNode based on the resourcesPerNode.
116-
resourcesPerNode := ptr.Deref(extractResourcePerNodeFromRuntime(info), corev1.ResourceRequirements{})
114+
resourcesPerNode := ptr.Deref(runtime.ExtractResourcePerNodeFromRuntime(info), corev1.ResourceRequirements{})
117115
if jobTrainer := trainJob.Spec.Trainer; jobTrainer != nil && jobTrainer.ResourcesPerNode != nil {
118116
resourcesPerNode = ptr.Deref(jobTrainer.ResourcesPerNode, corev1.ResourceRequirements{})
119117
}
120-
gpuQ := getNumGPUPerNode(&resourcesPerNode)
118+
gpuQ := runtime.GetNumGPUPerNode(&resourcesPerNode)
121119
// If numProcPerNode is "cpu" or no GPU is set in resource, we calculate numProcPerNode based on CPU.
122120
if numProcPerNode.String() == "cpu" || numProcPerNode.String() == "auto" && gpuQ == 0 {
123121
numProcPerNode = intstr.FromInt(max(1, getNumCPUPerNode(&resourcesPerNode)))
@@ -204,50 +202,3 @@ func getNumCPUPerNode(res *corev1.ResourceRequirements) int {
204202
}
205203
return int(requestCpuQ.Value())
206204
}
207-
208-
// getNumGPUPerNode returns the GPU count if found.
209-
func getNumGPUPerNode(res *corev1.ResourceRequirements) int {
210-
if res == nil {
211-
return 0
212-
}
213-
gpuQ := numGPU(res.Requests)
214-
if limitGpuQ := numGPU(res.Limits); gpuQ == 0 && limitGpuQ > 0 {
215-
gpuQ = limitGpuQ
216-
}
217-
return gpuQ
218-
}
219-
220-
func numGPU(resourcePerNode corev1.ResourceList) int {
221-
for resName, resQ := range resourcePerNode {
222-
if strings.Contains(strings.ToLower(resName.String()), "gpu") {
223-
return int(resQ.Value())
224-
}
225-
}
226-
return 0
227-
}
228-
229-
// extractResourcePerNodeFromRuntime extracts the resource per node from the Trainer Node.
230-
func extractResourcePerNodeFromRuntime(info *runtime.Info) *corev1.ResourceRequirements {
231-
if jobSetSpec, ok := runtime.TemplateSpecApply[jobsetv1alpha2ac.JobSetSpecApplyConfiguration](info); ok {
232-
for _, rJob := range jobSetSpec.ReplicatedJobs {
233-
if rJob.Name != nil && *rJob.Name == constants.Node || rJob.Template.Labels[constants.LabelTrainJobAncestor] == constants.AncestorTrainer {
234-
for _, container := range rJob.Template.Spec.Template.Spec.Containers {
235-
if container.Name != nil && *container.Name == constants.Node && container.Resources != nil {
236-
res := &corev1.ResourceRequirements{
237-
Limits: corev1.ResourceList{},
238-
Requests: corev1.ResourceList{},
239-
}
240-
if container.Resources.Limits != nil {
241-
res.Limits = *container.Resources.Limits
242-
}
243-
if container.Resources.Requests != nil {
244-
res.Requests = *container.Resources.Requests
245-
}
246-
return res
247-
}
248-
}
249-
}
250-
}
251-
}
252-
return nil
253-
}

‎pkg/runtime/framework/plugins/torch/torchtune.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,11 @@ func validateTorchTune(runtimeInfo *runtime.Info, newObj *trainer.TrainJob) (adm
5151

5252
numProcPerNodeRefPath := specPath.Child("trainer").Child("numProcPerNode")
5353
numProcPerNode := *newObj.Spec.Trainer.NumProcPerNode
54-
resourcesPerNode := ptr.Deref(extractResourcePerNodeFromRuntime(runtimeInfo), corev1.ResourceRequirements{})
54+
resourcesPerNode := ptr.Deref(runtime.ExtractResourcePerNodeFromRuntime(runtimeInfo), corev1.ResourceRequirements{})
5555
if jobTrainer := newObj.Spec.Trainer; jobTrainer != nil && jobTrainer.ResourcesPerNode != nil {
5656
resourcesPerNode = ptr.Deref(jobTrainer.ResourcesPerNode, corev1.ResourceRequirements{})
5757
}
58-
_, config := getRecipeAndConfig(numNodes, numProcPerNode, getNumGPUPerNode(&resourcesPerNode), newObj)
58+
_, config := getRecipeAndConfig(numNodes, numProcPerNode, runtime.GetNumGPUPerNode(&resourcesPerNode), newObj)
5959
if strings.Contains(config, constants.TorchTuneQLoRAFinetuneDistributedConfigSuffix) {
6060
if model == constants.TORCHTUNE_MODEL_QWEN2_5_1_5B {
6161
allErrs = append(allErrs, field.Invalid(runtimeRefNamePath, newObj.Spec.RuntimeRef.Name, fmt.Sprintf("QLoRA is not supported for %v model", model)))

‎pkg/runtime/runtime.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,17 @@ import (
2020
"iter"
2121
"maps"
2222
"slices"
23+
"strings"
2324

2425
corev1 "k8s.io/api/core/v1"
2526
"k8s.io/apimachinery/pkg/runtime/schema"
2627
corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
2728
resourcehelpers "k8s.io/component-helpers/resource"
2829
"k8s.io/utils/ptr"
30+
jobsetv1alpha2ac "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2"
2931

3032
trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1"
33+
"github.com/kubeflow/trainer/v2/pkg/constants"
3134
)
3235

3336
var (
@@ -240,3 +243,50 @@ func RuntimeRefToRuntimeRegistryKey(runtimeRef trainer.RuntimeRef) string {
240243
Kind: ptr.Deref(runtimeRef.Kind, ""),
241244
}.String()
242245
}
246+
247+
// ExtractResourcePerNodeFromRuntime extracts the Trainer resource per node from the Info object.
248+
func ExtractResourcePerNodeFromRuntime(info *Info) *corev1.ResourceRequirements {
249+
if jobSetSpec, ok := TemplateSpecApply[jobsetv1alpha2ac.JobSetSpecApplyConfiguration](info); ok {
250+
for _, rJob := range jobSetSpec.ReplicatedJobs {
251+
if rJob.Name != nil && *rJob.Name == constants.Node || rJob.Template.Labels[constants.LabelTrainJobAncestor] == constants.AncestorTrainer {
252+
for _, container := range rJob.Template.Spec.Template.Spec.Containers {
253+
if container.Name != nil && *container.Name == constants.Node && container.Resources != nil {
254+
res := &corev1.ResourceRequirements{
255+
Limits: corev1.ResourceList{},
256+
Requests: corev1.ResourceList{},
257+
}
258+
if container.Resources.Limits != nil {
259+
res.Limits = *container.Resources.Limits
260+
}
261+
if container.Resources.Requests != nil {
262+
res.Requests = *container.Resources.Requests
263+
}
264+
return res
265+
}
266+
}
267+
}
268+
}
269+
}
270+
return nil
271+
}
272+
273+
// GetNumGPUPerNode returns the GPU count if found in container resources.
274+
func GetNumGPUPerNode(res *corev1.ResourceRequirements) int {
275+
if res == nil {
276+
return 0
277+
}
278+
gpuQ := numGPU(res.Requests)
279+
if limitGpuQ := numGPU(res.Limits); gpuQ == 0 && limitGpuQ > 0 {
280+
gpuQ = limitGpuQ
281+
}
282+
return gpuQ
283+
}
284+
285+
func numGPU(resourcePerNode corev1.ResourceList) int {
286+
for resName, resQ := range resourcePerNode {
287+
if strings.Contains(strings.ToLower(resName.String()), "gpu") {
288+
return int(resQ.Value())
289+
}
290+
}
291+
return 0
292+
}

0 commit comments

Comments
 (0)