run_gemini_openvivqa_evaluation.sh

#!/bin/bash

# Bash script to run Gemini VivQAX evaluation with different reasoning efforts
# Usage: ./run_gemini_vivqax_evaluation.sh

set -e  # Exit on any error

echo "Starting Gemini VivQAX evaluation pipeline..."
echo "================================================"

# Define common variables
CUDA_DEVICE=2
CAPTION_NAME="gemini"
CAPTION_FILE="/home/vlai-gpt-oss/GPT_DAM_VQA/captions/gemini_openvivqa_captions.json"
DATASET="openvivqa"

# Function to check if GPU is available and ready
wait_for_gpu_availability() {
    local gpu_id=$1
    local check_interval=${2:-30}  # Default check every 30 seconds
    
    echo "Waiting for GPU $gpu_id to become available..."
    echo "Will check every ${check_interval} seconds until GPU is ready."
    echo "Press Ctrl+C to cancel."
    echo ""
    
    local attempt=1
    
    while true; do
        local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
        echo "[$timestamp] Attempt $attempt: Checking GPU $gpu_id..."
        
        # Check if nvidia-smi is available
        if ! command -v nvidia-smi &> /dev/null; then
            echo "  ❌ nvidia-smi not found. Waiting ${check_interval}s before retry..."
            sleep $check_interval
            ((attempt++))
            continue
        fi
        
        # Check if the specific GPU exists and get its status
        local gpu_info=$(nvidia-smi --id=$gpu_id --query-gpu=name,memory.used,memory.total,driver_version --format=csv,noheader,nounits 2>/dev/null)
        
        if [ $? -ne 0 ]; then
            echo "  ❌ GPU $gpu_id not found or not accessible."
            if [ $attempt -eq 1 ]; then
                echo "  Available GPUs:"
                nvidia-smi --list-gpus 2>/dev/null || echo "  No GPUs detected"
            fi
            echo "  Waiting ${check_interval}s before retry..."
            sleep $check_interval
            ((attempt++))
            continue
        fi
        
        local gpu_name=$(echo $gpu_info | cut -d',' -f1 | xargs)
        local used_memory=$(echo $gpu_info | cut -d',' -f2 | tr -d ' ')
        local total_memory=$(echo $gpu_info | cut -d',' -f3 | tr -d ' ')
        local driver_version=$(echo $gpu_info | cut -d',' -f4 | tr -d ' ')
        
        if [ -z "$used_memory" ] || [ -z "$total_memory" ] || [ "$total_memory" -eq 0 ]; then
            echo "  ❌ Could not parse GPU memory information. Waiting ${check_interval}s before retry..."
            sleep $check_interval
            ((attempt++))
            continue
        fi
        
        local usage_percent=$((used_memory * 100 / total_memory))
        
        echo "  📊 GPU $gpu_id: $gpu_name"
        echo "  📊 Driver: $driver_version"
        echo "  📊 Memory: ${used_memory}MB / ${total_memory}MB (${usage_percent}%)"
        
        if [ $usage_percent -gt 80 ]; then
            echo "  ⚠️  Memory usage too high (${usage_percent}%). Waiting for it to decrease..."
            echo "  Waiting ${check_interval}s before next check..."
            sleep $check_interval
            ((attempt++))
            continue
        else
            echo "  ✅ GPU $gpu_id is ready! Memory usage acceptable (${usage_percent}%)"
            echo ""
            return 0
        fi
    done
}

# Function to run evaluation and check for success
run_evaluation() {
    local eval_type=$1
    local reasoning_effort=$2
    local step_num=$3
    local total_steps=$4
    
    echo ""
    echo "Step $step_num/$total_steps: Running evaluation with $eval_type"
    if [ ! -z "$reasoning_effort" ]; then
        echo "Reasoning effort: $reasoning_effort"
    fi
    echo "----------------------------------------"
    
    if [ ! -z "$reasoning_effort" ]; then
        CUDA_VISIBLE_DEVICES=$CUDA_DEVICE python ./main.py \
            --eval $eval_type \
            --caption-name $CAPTION_NAME \
            --caption-file $CAPTION_FILE \
            --dataset $DATASET \
            --reasoning-effort $reasoning_effort \
            --eval-output evaluation_results_8192 
    else
        CUDA_VISIBLE_DEVICES=$CUDA_DEVICE python ./main.py \
            --eval $eval_type \
            --caption-name $CAPTION_NAME \
            --caption-file $CAPTION_FILE \
            --dataset $DATASET \
            --eval-output evaluation_results_8192 
    fi
    
    if [ $? -eq 0 ]; then
        echo "✓ Step $step_num completed successfully"
    else
        echo "✗ Step $step_num failed with error code $?"
        exit 1
    fi
}

# Check if caption file exists
if [ ! -f "$CAPTION_FILE" ]; then
    echo "Error: Caption file not found at $CAPTION_FILE"
    exit 1
fi

# Check if main.py exists
if [ ! -f "./main.py" ]; then
    echo "Error: main.py not found in current directory"
    exit 1
fi

echo "Using CUDA device: $CUDA_DEVICE"
echo "Caption file: $CAPTION_FILE"
echo "Dataset: $DATASET"
echo ""

# Check GPU availability - wait until ready
wait_for_gpu_availability $CUDA_DEVICE 30

# Step 1: Low reasoning effort
run_evaluation "gptoss" "low" 1 4

# Step 2: Medium reasoning effort
run_evaluation "gptoss" "medium" 2 4

# Step 3: High reasoning effort
run_evaluation "gptoss" "high" 3 4

# Step 4: Adaptive evaluation
run_evaluation "gptoss-adapt" "" 4 4

echo ""
echo "================================================"
echo "All evaluations completed successfully! ✓"
echo "Results should be available in the evaluation_results directory."