1 line
33 KiB
JSON
1 line
33 KiB
JSON
{"repo": "abnerai/medfm-robust", "files": {"VLM/run_vlm_perturbation_multi_gpu.sh": {"66615de683161bfb76609bb5abf343fdb05fc0bf": {"t": 1785745963.2575505, "n": 16166, "h": "b55b1e823337cf11", "c": "#!/bin/bash\n#SBATCH -o job.out\n#SBATCH --partition=a100\n#SBATCH -J vlm-final\n#SBATCH --nodes=1\n#SBATCH --ntasks-per-node=1\n#SBATCH --gres=gpu:3\n\n# ===================================================\n# Model Run Switches (true=run, false=skip)\n# ===================================================\nRUN_LLAVA=false\nRUN_MEDGEMMA=false\nRUN_MEDGEMMA15=false\nRUN_GEMINI=true # Only run Gemini Proxy\n\necho \"==========================================\"\necho \"Medical VLM Perturbation Evaluation - Multi-GPU Parallel Version\"\necho \"==========================================\"\necho \"Will run the following models:\"\n$RUN_LLAVA && echo \" - LLaVA-Med (GPU 0)\"\n$RUN_MEDGEMMA && echo \" - MedGemma (GPU 1)\"\n$RUN_MEDGEMMA15 && echo \" - MedGemma 1.5 (GPU 2)\"\n$RUN_GEMINI && echo \" - Gemini Proxy (API, no GPU required)\"\necho \"==========================================\"\n\n# watch tail logs/*.log\n\n# ============================================\n# Configuration Parameters\n# ============================================\nSAMPLE_NUM=100\nSEVERITIES=\"1,3,5\"\nSEED=42\nFINETUNE_EPOCHS=20\n\n# DeepSeek API Configuration\nDEEPSEEK_API_KEY=\"API\"\nDEEPSEEK_BASE_URL=\"https://api.deepseek.com\"\nUSE_DEEPSEEK=\"--use_deepseek\"\n\n# Gemini Proxy API Configuration\nGEMINI_PROXY_API_KEY=\"jobgpu_with_finetune.sh\"\nGEMINI_PROXY_BASE_URL=\"https://////////\"\nGEMINI_PROXY_MODEL=\"gemini-2.5-flash\"\n\n# Environment Paths\nENV_PERTURBATION=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/for_perb_except_seg\"\nENV_LLAVA=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/llava-med\"\nENV_MEDGEMMA=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/medgemma\"\nENV_MEDGEMMA15=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/medgemma\"\n\n# Model Paths\nLLAVA_MODEL_PATH=\"../models/llava-med-v1.5-mistral-7b\"\nMEDGEMMA_MODEL_PATH=\"/mnt/fast/nobackup/scratch4weeks/xxx/MedSAM/work_dir/medsam-vit-base/scripts/pipeline/Multi-scenario/Pipeline_for_All/models/medgemma-4b-it\"\nMEDGEMMA15_MODEL_PATH=\"/mnt/fast/nobackup/scratch4weeks/xxx/MedSAM/work_dir/medsam-vit-base/scripts/pipeline/Multi-scenario/Pipeline_for_All/models/medgemma-1.5-4b-it\"\n\n# Fine-tuning Output Directory\nFINETUNE_OUTPUT=\"./outputs/finetune\"\n\n# Script Paths\nSCRIPT_EVAL=\"./eval_vlm_perturbation.py\"\n\n# Log Directory\nLOG_DIR=\"./logs\"\nmkdir -p ${LOG_DIR}\n\n# ============================================\n# Define Evaluation Functions for Each Model\n# ============================================\n\nrun_llava_med() {\n echo \"[GPU 0] LLaVA-Med evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_LLAVA}\n\n export CUDA_VISIBLE_DEVICES=0\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Fine-tuning + Evaluation)\n FINETUNED_MODEL=\"${FINETUNE_OUTPUT}/llava-med/mecovqa_grounding/final\"\n if [ ! -d \"$FINETUNED_MODEL\" ]; then\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --finetune \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --finetune_output ${FINETUNE_OUTPUT} --epochs ${FINETUNE_EPOCHS}\n fi\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\t\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n echo \"[GPU 0] LLaVA-Med evaluation completed!\"\n}\n\nrun_medgemma() {\n echo \"[GPU 1] MedGemma evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_MEDGEMMA}\n\n export CUDA_VISIBLE_DEVICES=1\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Fine-tuning + Evaluation)\n FINETUNED_MODEL=\"${FINETUNE_OUTPUT}/medgemma/mecovqa_grounding/final\"\n if [ ! -d \"$FINETUNED_MODEL\" ]; then\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --finetune \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --finetune_output ${FINETUNE_OUTPUT} --epochs ${FINETUNE_EPOCHS}\n fi\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n echo \"[GPU 1] MedGemma evaluation completed!\"\n}\n\nrun_medgemma15() {\n echo \"[GPU 2] MedGemma 1.5 evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_MEDGEMMA15}\n\n export CUDA_VISIBLE_DEVICES=2\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Fine-tuning + Evaluation)\n FINETUNED_MODEL=\"${FINETUNE_OUTPUT}/medgemma15/mecovqa_grounding/final\"\n if [ ! -d \"$FINETUNED_MODEL\" ]; then\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --finetune \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --finetune_output ${FINETUNE_OUTPUT} --epochs ${FINETUNE_EPOCHS}\n fi\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\t\n echo \"[GPU 2] MedGemma 1.5 evaluation completed!\"\n}\n\nrun_gemini_proxy() {\n echo \"[API] Gemini Proxy evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_MEDGEMMA} # Use environment with torch\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Closed-source API does not support fine-tuning, direct zero-shot evaluation)\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n echo \"[API] Gemini Proxy evaluation completed!\"\n}\n\n# ============================================\n# Execute Selected Models in Parallel\n# ============================================\necho \"\"\necho \"==========================================\"\necho \"Starting parallel evaluation...\"\necho \"==========================================\"\necho \"\"\n\n# Record started processes\nPIDS=\"\"\n\n# Start corresponding models based on switches\nif $RUN_LLAVA; then\n run_llava_med > ${LOG_DIR}/llava_med.log 2>&1 &\n PID_LLAVA=$!\n PIDS=\"$PIDS $PID_LLAVA\"\n echo \"LLaVA-Med started (PID: $PID_LLAVA, GPU: 0)\"\nfi\n\nif $RUN_MEDGEMMA; then\n run_medgemma > ${LOG_DIR}/medgemma.log 2>&1 &\n PID_MEDGEMMA=$!\n PIDS=\"$PIDS $PID_MEDGEMMA\"\n echo \"MedGemma started (PID: $PID_MEDGEMMA, GPU: 1)\"\nfi\n\nif $RUN_MEDGEMMA15; then\n run_medgemma15 > ${LOG_DIR}/medgemma15.log 2>&1 &\n PID_MEDGEMMA15=$!\n PIDS=\"$PIDS $PID_MEDGEMMA15\"\n echo \"MedGemma 1.5 started (PID: $PID_MEDGEMMA15, GPU: 2)\"\nfi\n\nif $RUN_GEMINI; then\n run_gemini_proxy > ${LOG_DIR}/gemini_proxy.log 2>&1 &\n PID_GEMINI=$!\n PIDS=\"$PIDS $PID_GEMINI\"\n echo \"Gemini Proxy started (PID: $PID_GEMINI, API call, no GPU required)\"\nfi\n\necho \"\"\necho \"Started processes: $PIDS\"\necho \"View logs: tail -f ${LOG_DIR}/*.log\"\necho \"\"\n\n# Wait for all background processes to complete\nif $RUN_LLAVA; then\n wait $PID_LLAVA\n STATUS_LLAVA=$?\n echo \"LLaVA-Med completed (exit code: $STATUS_LLAVA)\"\nfi\n\nif $RUN_MEDGEMMA; then\n wait $PID_MEDGEMMA\n STATUS_MEDGEMMA=$?\n echo \"MedGemma completed (exit code: $STATUS_MEDGEMMA)\"\nfi\n\nif $RUN_MEDGEMMA15; then\n wait $PID_MEDGEMMA15\n STATUS_MEDGEMMA15=$?\n echo \"MedGemma 1.5 completed (exit code: $STATUS_MEDGEMMA15)\"\nfi\n\nif $RUN_GEMINI; then\n wait $PID_GEMINI\n STATUS_GEMINI=$?\n echo \"Gemini Proxy completed (exit code: $STATUS_GEMINI)\"\nfi\n\n# ============================================\n# Merge Results and Generate Paper Tables\n# ============================================\necho \"\"\necho \"==========================================\"\necho \"Merging results and generating paper tables\"\necho \"==========================================\"\n\nsource ~/miniconda3/bin/activate\nconda activate ${ENV_LLAVA}\n\nSCRIPT_MERGE=\"./merge_results_to_table.py\"\nOUTPUT_DIR=\"./outputs/vlm_perturbation_results\"\nTABLE_DIR=\"./outputs/paper_tables\"\nmkdir -p ${TABLE_DIR}\n\nRESULT_COUNT=$(find ${OUTPUT_DIR} -name \"results_*.json\" -type f 2>/dev/null | wc -l)\n\nif [ \"$RESULT_COUNT\" -gt 0 ]; then\n echo \"Found ${RESULT_COUNT} result files, generating paper tables...\"\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} --task vqa \\\n --output ${TABLE_DIR}/table_vqa_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_vqa_all_models.csv\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} --task caption \\\n --output ${TABLE_DIR}/table_caption_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_caption_all_models.csv\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} --task grounding \\\n --output ${TABLE_DIR}/table_grounding_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_grounding_all_models.csv\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} \\\n --output ${TABLE_DIR}/table_all_tasks_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_all_tasks_all_models.csv\n\n echo \"Paper tables saved to: ${TABLE_DIR}/\"\nfi\n\necho \"\"\necho \"==========================================\"\necho \"All completed!\"\necho \"==========================================\"\necho \"\"\necho \"Log files:\"\necho \" ${LOG_DIR}/llava_med.log\"\necho \" ${LOG_DIR}/medgemma.log\"\necho \" ${LOG_DIR}/medgemma15.log\"\necho \" ${LOG_DIR}/gemini_proxy.log\"\necho \"\"\n"}, "31affe2ab0945613e21b0f90485161d594948a2c": {"t": 1785758329.4692953, "n": 16166, "h": "b55b1e823337cf11", "c": "#!/bin/bash\n#SBATCH -o job.out\n#SBATCH --partition=a100\n#SBATCH -J vlm-final\n#SBATCH --nodes=1\n#SBATCH --ntasks-per-node=1\n#SBATCH --gres=gpu:3\n\n# ===================================================\n# Model Run Switches (true=run, false=skip)\n# ===================================================\nRUN_LLAVA=false\nRUN_MEDGEMMA=false\nRUN_MEDGEMMA15=false\nRUN_GEMINI=true # Only run Gemini Proxy\n\necho \"==========================================\"\necho \"Medical VLM Perturbation Evaluation - Multi-GPU Parallel Version\"\necho \"==========================================\"\necho \"Will run the following models:\"\n$RUN_LLAVA && echo \" - LLaVA-Med (GPU 0)\"\n$RUN_MEDGEMMA && echo \" - MedGemma (GPU 1)\"\n$RUN_MEDGEMMA15 && echo \" - MedGemma 1.5 (GPU 2)\"\n$RUN_GEMINI && echo \" - Gemini Proxy (API, no GPU required)\"\necho \"==========================================\"\n\n# watch tail logs/*.log\n\n# ============================================\n# Configuration Parameters\n# ============================================\nSAMPLE_NUM=100\nSEVERITIES=\"1,3,5\"\nSEED=42\nFINETUNE_EPOCHS=20\n\n# DeepSeek API Configuration\nDEEPSEEK_API_KEY=\"API\"\nDEEPSEEK_BASE_URL=\"https://api.deepseek.com\"\nUSE_DEEPSEEK=\"--use_deepseek\"\n\n# Gemini Proxy API Configuration\nGEMINI_PROXY_API_KEY=\"jobgpu_with_finetune.sh\"\nGEMINI_PROXY_BASE_URL=\"https://////////\"\nGEMINI_PROXY_MODEL=\"gemini-2.5-flash\"\n\n# Environment Paths\nENV_PERTURBATION=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/for_perb_except_seg\"\nENV_LLAVA=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/llava-med\"\nENV_MEDGEMMA=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/medgemma\"\nENV_MEDGEMMA15=\"/mnt/fast/nobackup/scratch4weeks/xxx/envs/medgemma\"\n\n# Model Paths\nLLAVA_MODEL_PATH=\"../models/llava-med-v1.5-mistral-7b\"\nMEDGEMMA_MODEL_PATH=\"/mnt/fast/nobackup/scratch4weeks/xxx/MedSAM/work_dir/medsam-vit-base/scripts/pipeline/Multi-scenario/Pipeline_for_All/models/medgemma-4b-it\"\nMEDGEMMA15_MODEL_PATH=\"/mnt/fast/nobackup/scratch4weeks/xxx/MedSAM/work_dir/medsam-vit-base/scripts/pipeline/Multi-scenario/Pipeline_for_All/models/medgemma-1.5-4b-it\"\n\n# Fine-tuning Output Directory\nFINETUNE_OUTPUT=\"./outputs/finetune\"\n\n# Script Paths\nSCRIPT_EVAL=\"./eval_vlm_perturbation.py\"\n\n# Log Directory\nLOG_DIR=\"./logs\"\nmkdir -p ${LOG_DIR}\n\n# ============================================\n# Define Evaluation Functions for Each Model\n# ============================================\n\nrun_llava_med() {\n echo \"[GPU 0] LLaVA-Med evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_LLAVA}\n\n export CUDA_VISIBLE_DEVICES=0\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Fine-tuning + Evaluation)\n FINETUNED_MODEL=\"${FINETUNE_OUTPUT}/llava-med/mecovqa_grounding/final\"\n if [ ! -d \"$FINETUNED_MODEL\" ]; then\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --finetune \\\n --model_type llava-med --model_path ${LLAVA_MODEL_PATH} \\\n --finetune_output ${FINETUNE_OUTPUT} --epochs ${FINETUNE_EPOCHS}\n fi\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\t\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type llava-med --model_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n echo \"[GPU 0] LLaVA-Med evaluation completed!\"\n}\n\nrun_medgemma() {\n echo \"[GPU 1] MedGemma evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_MEDGEMMA}\n\n export CUDA_VISIBLE_DEVICES=1\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Fine-tuning + Evaluation)\n FINETUNED_MODEL=\"${FINETUNE_OUTPUT}/medgemma/mecovqa_grounding/final\"\n if [ ! -d \"$FINETUNED_MODEL\" ]; then\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --finetune \\\n --model_type medgemma --medgemma_path ${MEDGEMMA_MODEL_PATH} \\\n --finetune_output ${FINETUNE_OUTPUT} --epochs ${FINETUNE_EPOCHS}\n fi\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma --medgemma_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n echo \"[GPU 1] MedGemma evaluation completed!\"\n}\n\nrun_medgemma15() {\n echo \"[GPU 2] MedGemma 1.5 evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_MEDGEMMA15}\n\n export CUDA_VISIBLE_DEVICES=2\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Fine-tuning + Evaluation)\n FINETUNED_MODEL=\"${FINETUNE_OUTPUT}/medgemma15/mecovqa_grounding/final\"\n if [ ! -d \"$FINETUNED_MODEL\" ]; then\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --finetune \\\n --model_type medgemma15 --medgemma15_path ${MEDGEMMA15_MODEL_PATH} \\\n --finetune_output ${FINETUNE_OUTPUT} --epochs ${FINETUNE_EPOCHS}\n fi\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type medgemma15 --medgemma15_path ${FINETUNED_MODEL} \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\t\n echo \"[GPU 2] MedGemma 1.5 evaluation completed!\"\n}\n\nrun_gemini_proxy() {\n echo \"[API] Gemini Proxy evaluation started...\"\n\n source ~/miniconda3/bin/activate\n conda activate ${ENV_MEDGEMMA} # Use environment with torch\n\n # OmniMedVQA VQA\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset omnimedvqa --task vqa --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # ROCO Caption\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset roco --task caption --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n # MeCoVQA Grounding (Closed-source API does not support fine-tuning, direct zero-shot evaluation)\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK}\n\n python ${SCRIPT_EVAL} --dataset mecovqa --task grounding --sample_num ${SAMPLE_NUM} \\\n --model_type gemini-proxy \\\n --gemini_proxy_api_key \"${GEMINI_PROXY_API_KEY}\" \\\n --gemini_proxy_base_url \"${GEMINI_PROXY_BASE_URL}\" \\\n --gemini_proxy_model \"${GEMINI_PROXY_MODEL}\" \\\n --deepseek_api_key \"${DEEPSEEK_API_KEY}\" --deepseek_base_url \"${DEEPSEEK_BASE_URL}\" ${USE_DEEPSEEK} \\\n --perturbed --all_perturbations\n\n echo \"[API] Gemini Proxy evaluation completed!\"\n}\n\n# ============================================\n# Execute Selected Models in Parallel\n# ============================================\necho \"\"\necho \"==========================================\"\necho \"Starting parallel evaluation...\"\necho \"==========================================\"\necho \"\"\n\n# Record started processes\nPIDS=\"\"\n\n# Start corresponding models based on switches\nif $RUN_LLAVA; then\n run_llava_med > ${LOG_DIR}/llava_med.log 2>&1 &\n PID_LLAVA=$!\n PIDS=\"$PIDS $PID_LLAVA\"\n echo \"LLaVA-Med started (PID: $PID_LLAVA, GPU: 0)\"\nfi\n\nif $RUN_MEDGEMMA; then\n run_medgemma > ${LOG_DIR}/medgemma.log 2>&1 &\n PID_MEDGEMMA=$!\n PIDS=\"$PIDS $PID_MEDGEMMA\"\n echo \"MedGemma started (PID: $PID_MEDGEMMA, GPU: 1)\"\nfi\n\nif $RUN_MEDGEMMA15; then\n run_medgemma15 > ${LOG_DIR}/medgemma15.log 2>&1 &\n PID_MEDGEMMA15=$!\n PIDS=\"$PIDS $PID_MEDGEMMA15\"\n echo \"MedGemma 1.5 started (PID: $PID_MEDGEMMA15, GPU: 2)\"\nfi\n\nif $RUN_GEMINI; then\n run_gemini_proxy > ${LOG_DIR}/gemini_proxy.log 2>&1 &\n PID_GEMINI=$!\n PIDS=\"$PIDS $PID_GEMINI\"\n echo \"Gemini Proxy started (PID: $PID_GEMINI, API call, no GPU required)\"\nfi\n\necho \"\"\necho \"Started processes: $PIDS\"\necho \"View logs: tail -f ${LOG_DIR}/*.log\"\necho \"\"\n\n# Wait for all background processes to complete\nif $RUN_LLAVA; then\n wait $PID_LLAVA\n STATUS_LLAVA=$?\n echo \"LLaVA-Med completed (exit code: $STATUS_LLAVA)\"\nfi\n\nif $RUN_MEDGEMMA; then\n wait $PID_MEDGEMMA\n STATUS_MEDGEMMA=$?\n echo \"MedGemma completed (exit code: $STATUS_MEDGEMMA)\"\nfi\n\nif $RUN_MEDGEMMA15; then\n wait $PID_MEDGEMMA15\n STATUS_MEDGEMMA15=$?\n echo \"MedGemma 1.5 completed (exit code: $STATUS_MEDGEMMA15)\"\nfi\n\nif $RUN_GEMINI; then\n wait $PID_GEMINI\n STATUS_GEMINI=$?\n echo \"Gemini Proxy completed (exit code: $STATUS_GEMINI)\"\nfi\n\n# ============================================\n# Merge Results and Generate Paper Tables\n# ============================================\necho \"\"\necho \"==========================================\"\necho \"Merging results and generating paper tables\"\necho \"==========================================\"\n\nsource ~/miniconda3/bin/activate\nconda activate ${ENV_LLAVA}\n\nSCRIPT_MERGE=\"./merge_results_to_table.py\"\nOUTPUT_DIR=\"./outputs/vlm_perturbation_results\"\nTABLE_DIR=\"./outputs/paper_tables\"\nmkdir -p ${TABLE_DIR}\n\nRESULT_COUNT=$(find ${OUTPUT_DIR} -name \"results_*.json\" -type f 2>/dev/null | wc -l)\n\nif [ \"$RESULT_COUNT\" -gt 0 ]; then\n echo \"Found ${RESULT_COUNT} result files, generating paper tables...\"\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} --task vqa \\\n --output ${TABLE_DIR}/table_vqa_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_vqa_all_models.csv\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} --task caption \\\n --output ${TABLE_DIR}/table_caption_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_caption_all_models.csv\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} --task grounding \\\n --output ${TABLE_DIR}/table_grounding_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_grounding_all_models.csv\n\n python ${SCRIPT_MERGE} --input_dir ${OUTPUT_DIR} \\\n --output ${TABLE_DIR}/table_all_tasks_all_models.xlsx \\\n --output_csv ${TABLE_DIR}/table_all_tasks_all_models.csv\n\n echo \"Paper tables saved to: ${TABLE_DIR}/\"\nfi\n\necho \"\"\necho \"==========================================\"\necho \"All completed!\"\necho \"==========================================\"\necho \"\"\necho \"Log files:\"\necho \" ${LOG_DIR}/llava_med.log\"\necho \" ${LOG_DIR}/medgemma.log\"\necho \" ${LOG_DIR}/medgemma15.log\"\necho \" ${LOG_DIR}/gemini_proxy.log\"\necho \"\"\n"}}}} |