Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion lmms_eval/api/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,6 @@ def perplexity_fn(items): # This is a passthrough function
return items



def levenshtein_distance(s1, s2):
if len(s1) > len(s2):
s1, s2 = s2, s1
Expand Down
1 change: 0 additions & 1 deletion lmms_eval/api/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
"""
pass


# TODO: Add an optional max length
@abc.abstractmethod
def generate_until(self, requests) -> List[str]:
Expand Down
4 changes: 3 additions & 1 deletion lmms_eval/api/samplers.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ def get_context(self, doc, num_fewshot):
+ (
str(self.doc_to_target(doc)[0])
if type(self.doc_to_target(doc)) is list
else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
else self.doc_to_target(doc)
if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str)
else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
)
for doc in selected_docs
]
Expand Down
2 changes: 1 addition & 1 deletion lmms_eval/api/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,7 @@ def download(self, dataset_kwargs=None) -> None:
download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
**dataset_kwargs if dataset_kwargs is not None else {},
)
self.dataset_no_image = datasets.load_dataset(
self.dataset_no_image = datasets.load_dataset(
path=self.DATASET_PATH,
name=self.DATASET_NAME,
download_mode=datasets.DownloadMode.REUSE_DATASET_IF_EXISTS,
Expand Down
32 changes: 19 additions & 13 deletions lmms_eval/models/__init__.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
import os

try:
# enabling faster model download
from .llava import Llava
from .qwen_vl import Qwen_VL
from .fuyu import Fuyu
from .gpt4v import GPT4V
from .instructblip import InstructBLIP
from .minicpm_v import MiniCPM_V
import hf_transfer

os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
except ImportError:
pass
AVAILABLE_MODELS = {
"llava": "Llava",
"qwen_vl": "Qwen_VL",
"fuyu": "Fuyu",
"gpt4v": "GPT4V",
"instructblip": "InstructBLIP",
"minicpm_v": "MiniCPM_V",
}

for model_name, model_class in AVAILABLE_MODELS.items():
try:
exec(f"from .{model_name} import {model_class}")
except ImportError:
pass


import hf_transfer

os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
2 changes: 0 additions & 2 deletions lmms_eval/models/fuyu.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,8 +253,6 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
pbar.close()
return res



def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None) -> List[int]:
""" """
add_special_tokens = False if add_special_tokens is None else add_special_tokens
Expand Down
2 changes: 0 additions & 2 deletions lmms_eval/models/gpt4v.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,5 +127,3 @@ def generate_until(self, requests) -> List[str]:
def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
# TODO
assert False, "GPT4V not support"


10 changes: 3 additions & 7 deletions lmms_eval/models/instructblip.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,26 +44,22 @@ def __init__(
self._device = torch.device(f"cuda:{accelerator.local_process_index}")
else:
self._device = device
self._model = InstructBlipForConditionalGeneration.from_pretrained(pretrained,device_map=self._device)
self._model = InstructBlipForConditionalGeneration.from_pretrained(pretrained, device_map=self._device)
self._image_processor = InstructBlipProcessor.from_pretrained(pretrained)
self._tokenizer = self._image_processor.tokenizer
self._config = self._model.config
self.model.eval()
self.model.tie_weights()
self.batch_size_per_gpu = int(batch_size)
if accelerator.num_processes > 1:
assert accelerator.distributed_type in [
DistributedType.FSDP,
DistributedType.MULTI_GPU,
DistributedType.DEEPSPEED
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
if accelerator.distributed_type == DistributedType.DEEPSPEED:
kwargs = {
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
"train_batch_size" : self.batch_size_per_gpu * accelerator.num_processes,
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
}
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
Expand Down
8 changes: 2 additions & 6 deletions lmms_eval/models/llava.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,14 @@ def __init__(
self.truncate_context = truncate_context
# assert self.batch_size_per_gpu == 1, "Llava currently does not support batched generation. See https://github.com/haotian-liu/LLaVA/issues/754. HF Llava also has this issue."
if accelerator.num_processes > 1:
assert accelerator.distributed_type in [
DistributedType.FSDP,
DistributedType.MULTI_GPU,
DistributedType.DEEPSPEED
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
if accelerator.distributed_type == DistributedType.DEEPSPEED:
kwargs = {
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
"train_batch_size" : self.batch_size_per_gpu * accelerator.num_processes,
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
}
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
Expand Down
10 changes: 3 additions & 7 deletions lmms_eval/models/minicpm_v.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,14 @@ def __init__(
self.model.tie_weights()
self.batch_size_per_gpu = int(batch_size)
if accelerator.num_processes > 1:
assert accelerator.distributed_type in [
DistributedType.FSDP,
DistributedType.MULTI_GPU,
DistributedType.DEEPSPEED
], "Unsupported distributed type provided. Only DDP and FSDP are supported."
assert accelerator.distributed_type in [DistributedType.FSDP, DistributedType.MULTI_GPU, DistributedType.DEEPSPEED], "Unsupported distributed type provided. Only DDP and FSDP are supported."
# If you want to use DistributedType.DEEPSPEED, you have to run accelerate config before using the model
# Also, you have to select zero stage 0 (equivalent to DDP) in order to make the prepare model works
# I tried to set different parameters in the kwargs to let default zero 2 stage works, but it didn't work.
if accelerator.distributed_type == DistributedType.DEEPSPEED:
kwargs = {
"train_micro_batch_size_per_gpu": self.batch_size_per_gpu,
"train_batch_size" : self.batch_size_per_gpu * accelerator.num_processes,
"train_batch_size": self.batch_size_per_gpu * accelerator.num_processes,
}
AcceleratorState().deepspeed_plugin.deepspeed_config_process(must_match=True, **kwargs)
eval_logger.info("Detected that you are using DistributedType.DEEPSPEED. Make sure you run `accelerate config` and set zero stage to 0")
Expand Down Expand Up @@ -189,7 +185,7 @@ def _collate(x):
if "<image>" in context:
# minicpm does not expect the <image> tag
context = context.replace("<image>", "")
msgs = [{'role': 'user', 'content': context}]
msgs = [{"role": "user", "content": context}]

gen_kwargs["image_sizes"] = [visuals[idx].size for idx in range(len(visuals))]
if "max_new_tokens" not in gen_kwargs:
Expand Down
2 changes: 0 additions & 2 deletions lmms_eval/models/qwen_vl.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,8 +174,6 @@ def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
pbar.close()
return res



def flatten(self, input):
new_list = []
for i in input:
Expand Down
2 changes: 1 addition & 1 deletion lmms_eval/tasks/cmmmu/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def cmmmu_process_test_results_for_submission(doc, results):

def cmmmu_test_aggregate_results_for_submission(results, args):
file = generate_submission_file("cmmmu_test_for_submission.jsonl", args)
with open(file, "w", encoding='utf8') as f:
with open(file, "w", encoding="utf8") as f:
for result in results:
json.dump(result, f, ensure_ascii=False)
f.write("\n")
Expand Down
5 changes: 3 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
name = "lmms_eval"
version = "0.1.0"
authors = [
{ name = "LMMMs-Lab Evaluation Team", email = "lmms_eval@gmail.com" },
{ name = "LMMMs-Lab Evaluation Team", email = "lmms_eval@outlook.com" },
]
description = "A framework for evaluating large multi-modality language models"
readme = "README.md"
Expand Down Expand Up @@ -53,6 +53,7 @@ dependencies = [
"transformers-stream-generator",
"tiktoken",
"pre-commit",
"pydantic",
]

[tool.setuptools.packages.find]
Expand All @@ -66,5 +67,5 @@ lmms-eval = "lmms_eval.__main__:cli_evaluate"
lmms_eval = "lmms_eval.__main__:cli_evaluate"

[project.urls]
Homepage = "https://github.com/EvolvingLMMs-Lab/lmms-eval"
Homepage = "https://lmms-lab.github.io/lmms-eval-blog/"
Repository = "https://github.com/EvolvingLMMs-Lab/lmms-eval"