Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions lmms_eval/evaluator.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import os
import time
import random
import itertools
import json
Expand Down Expand Up @@ -428,6 +430,15 @@ def evaluate(
# Ensure all ranks wait for rank 0 to finish aggregation
torch.distributed.barrier()

# Synchronize processes with a temp file in case the evluation metric requires gpus
# TODO: fix barriers' taking up gpu computation
os.makedirs(cli_args.output_path, exist_ok=True)
if os.path.exists(f"{cli_args.output_path}/rank0_metric_eval_done.txt"):
try:
os.remove(f"{cli_args.output_path}/rank0_metric_eval_done.txt")
except:
pass

if lm.rank == 0:
### Get task ordering for correct sample-wide aggregation
group_to_task = {}
Expand Down Expand Up @@ -629,7 +640,12 @@ def print_tasks(task_hierarchy, task_order, task_version, task_group_alias):
if log_samples:
results_dict["samples"] = dict(samples)

return results_dict

with open(f"{cli_args.output_path}/rank0_metric_eval_done.txt", 'w') as f:
f.write('rank0 metric eval done')
else:
return None
results_dict = None
if int(os.environ.get("RANK", 0)) != 0:
while not os.path.exists(f"{cli_args.output_path}/rank0_metric_eval_done.txt"):
time.sleep(1)

return results_dict