evaluate_qwen.sh

#!/bin/bash

# Exit on any error
set -e

# =================== Configuration Variables ===================
MODEL_NAME="Qwen2.5-VL-7B"
NUM_GPUS=1
OUTPUT_DIR="eval_results"

CATEGORY="all"

PREDICTION_PATH="${OUTPUT_DIR}/${MODEL_NAME}"

OPENAI_API_KEY=""  # Replace with your actual key or use environment variable
# ==============================================================

echo "Starting evaluation pipeline..."

# Derive prediction output path


# Ensure output directory exists
mkdir -p "${OUTPUT_DIR}"
echo "Step 1: Running model evaluation with Accelerate"s
accelerate launch --num_processes ${NUM_GPUS} evaluation/shotvl/evaluate_qwen.py --model ${MODEL_NAME} --reasoning --output-dir ${OUTPUT_DIR} --category "${CATEGORY}"

echo "Model evaluation completed. Predictions saved to: ${PREDICTION_PATH}"

# echo "Step 2: Calculating scores using OpenAI API"
# OPENAI_API_KEY=${OPENAI_API_KEY} python evaluation/calculate_scores.py --prediction_path ${PREDICTION_PATH} --category "${CATEGORY}"

# echo "Score calculation completed."
# echo "All steps finished successfully."