1 line
14 KiB
JSON
1 line
14 KiB
JSON
{"repo": "anonyupload/egogazevqa", "files": {"generate_tool/causal.py": {"a9f1c5d5e006988f5a6b57d4a53d9a3716a562b8": {"t": 1785846856.902, "n": 13500, "h": "1bc33913516e5968", "c": "import os\nimport json\nimport csv\nfrom openai import OpenAI\nimport base64\nimport argparse\n\n\ndef group_frames_and_generate_text(image_folder, json_file, target_group_index, group_size=9):\n \"\"\"\n Groups images from a folder, retrieves related narration and gaze info from JSON,\n and generates text pairs in the format \"Frame X [Caption; gaze info]\".\n \n :param image_folder: Path to the folder containing image files.\n :param json_file: Path to the JSON file containing narration data.\n :param group_size: Number of images per group (default: 9).\n \"\"\"\n try:\n # Load narration JSON\n with open(json_file, \"r\") as f:\n narrations = json.load(f)\n\n # Retrieve the video ID from the folder name\n video_id = os.path.basename(image_folder)\n print(f\"Video_id: {video_id}\\nCurrent group:{target_group_index}\")\n \n # Ensure the JSON contains data for this video ID\n if video_id not in narrations:\n raise ValueError(f\"Video ID {video_id} not found in the JSON file.\")\n \n narration_data = narrations[video_id][\"narration_pass_1\"][\"narrations\"]\n\n # Create a mapping of frame numbers to their narration and gaze info\n frame_to_narration = {\n item[\"timestamp_frame\"]: {\n \"narration_text\": item[\"narration_text\"],\n \"gaze_info\": item.get(\"gaze_info\", {})\n }\n for item in narration_data\n }\n\n # List and sort image files\n image_files = sorted(\n [f for f in os.listdir(image_folder) if f.endswith(('.jpg'))],\n key=lambda x: int(os.path.splitext(x)[0])\n )\n\n # Group images and generate text pairs\n text_pairs = []\n for i in range(0, len(image_files), group_size):\n group_index = i // group_size # 当前分组的索引\n if group_index == target_group_index: # 只处理目标分组\n group = image_files[i:i + group_size]\n for j, image_file in enumerate(group):\n frame_number = int(os.path.splitext(image_file)[0]) # 从文件名提取帧号\n if frame_number in frame_to_narration:\n narration_text = frame_to_narration[frame_number][\"narration_text\"]\n gaze_info = frame_to_narration[frame_number][\"gaze_info\"]\n gaze_text = f\"Gaze:({gaze_info.get('gaze_x', 0)},{gaze_info.get('gaze_y', 0)})\"\n text_pair = f\"Frame {j + 1}: {narration_text}; {gaze_text}\"\n text_pairs.append(text_pair)\n else:\n print(f\"No narration data for frame {frame_number}, skipping.\")\n \n return text_pairs, group\n \n except Exception as e:\n print(f\"Error: {e}\")\n \n\ndef encode_images_from_folder(folder_path, group_id):\n \"\"\"\n Encode all image files in a folder to Base64 and return them as a list of dictionaries.\n \n :param folder_path: Path to the folder containing images.\n :return: List of dictionaries with Base64 encoded images in the required format.\n \"\"\"\n image_data_list = []\n # Sort files to ensure correct order\n \n for image_file in group_id:\n image_path = os.path.join(folder_path, image_file)\n with open(image_path, \"rb\") as img_file:\n base64_image = base64.b64encode(img_file.read()).decode(\"utf-8\")\n image_data_list.append(f\"data:image/jpeg;base64,{base64_image}\")\n return image_data_list\n\nimport json\n\ndef append_to_json(target_group_index, group_id, caption, completion_content, output_path):\n \"\"\"\n Append new data to an existing JSON file or create a new one.\n\n :param video_id: ID of the video being processed.\n :param target_group_index: Index of the current target group.\n :param group_id: ID of the group being processed.\n :param completion_content: Content generated by the completion model.\n :param output_path: Path to save the JSON file.\n \"\"\"\n\n new_entry = {\n \"current_group\": target_group_index,\n \"group_id\": group_id,\n \"caption\": caption,\n \"completion_content\": completion_content\n }\n\n\n if os.path.exists(output_path):\n with open(output_path, \"r\") as json_file:\n existing_data = json.load(json_file)\n else:\n existing_data = []\n\n\n existing_data.append(new_entry)\n\n\n with open(output_path, \"w\") as json_file:\n json.dump(existing_data, json_file, indent=4)\n\n print(f\"New entry appended to {output_path}\")\n\ndef save_completion_to_csv(video_id, group_id, completion_content, output_csv):\n \"\"\"\n Extract information from completion content and save to CSV file.\n\n :param video_id: ID of the video.\n :param group_id: ID of the group.\n :param completion_content: Completion content to parse.\n :param output_csv: Path to the output CSV file.\n \"\"\"\n \n lines = completion_content.split(\"\\n\")\n question = \"\"\n answer_options = []\n correct_answer = \"\"\n \n\n is_answer_section = False\n\n for i, line in enumerate(lines):\n line = line.strip()\n if line.startswith(\"### Question\"):\n \n for j in range(i + 1, len(lines)):\n next_line = lines[j].strip()\n if next_line.startswith(\"### Answer Options:\"): \n break\n if \"what\" in next_line.lower() or \"where\" in next_line.lower() or \"when\" in next_line.lower() or \"why\" in next_line.lower() or \"which\" in next_line.lower() or \"how\" in next_line.lower(): # 查找包含 \"question\" 的行\n question = next_line.strip()\n elif line.startswith(\"### Answer Options:\"):\n is_answer_section = True \n elif line.startswith(\"### Correct Answer:\"):\n is_answer_section = False \n correct_answer = lines[i + 1][0].strip() \n elif is_answer_section and (line.startswith(\"A:\") or line.startswith(\"B:\") or line.startswith(\"C:\") or line.startswith(\"D:\") or line.startswith(\"E:\")):\n answer_options.append(line) \n \n file_exists = os.path.exists(output_csv)\n \n data = [\n {\n \"video_id\": video_id,\n \"group_id\": \"\\n\".join(group_id),\n \"Question\": question,\n \"Answer Options\": \"\\n\".join(answer_options),\n \"Correct Answer\": correct_answer\n }\n ]\n\n\n with open(output_csv, \"a\", newline=\"\", encoding=\"utf-8\") as csvfile:\n fieldnames = [\"video_id\", \"group_id\", \"Question\", \"Answer Options\", \"Correct Answer\"]\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n\n \n if not file_exists:\n writer.writeheader()\n\n writer.writerows(data)\n \n print(f\"Data appended to {output_csv}\")\n\ndef main():\n parser = argparse.ArgumentParser(description=\" video_id and target_group_index \")\n\n parser.add_argument('--video_id', type=str, required=True)\n parser.add_argument('--target_index', type=int, required=True)\n\n args = parser.parse_args()\n\n video_id = args.video_id\n target_group_index = args.target_index\n\n \n image_folder = f\"/home/pty_ssd/output0207/{video_id}\"\n json_file = \"/home/pty_ssd/narration_with_gaze_600s.json\"\n output_json = f\"/home/pty_ssd/Qwen2-VL/QA_benchmark0207/reasoning/json/{video_id}.json\"\n output_csv = f\"/home/pty_ssd/Qwen2-VL/QA_benchmark0207/reasoning/csv_v3/{video_id}.csv\"\n\n\n text_pairs, group_id = group_frames_and_generate_text(image_folder, json_file, target_group_index)\n print(f\"Group_id: {group_id}\")\n image_data_list = encode_images_from_folder(image_folder, group_id)\n\n joint_text = \"\\n\".join(text_pairs)\n\n try:\n client = OpenAI(\n api_key='sk-',\n base_url=\"https://dashscope.aliyuncs.com/compatible-mode/v1\",\n )\n\n\n completion = client.chat.completions.create(\n model=\"qwen-vl-max-latest\",\n messages=[\n {\n \"role\": \"system\",\n \"content\":[\n {\"type\": \"text\",\n \"text\": \"You are an expert in gaze-based event causal understanding within ego-centric video environments. Your task is to design comprehensive Gaze-Informed Causal Reasoning ego-centric VideoQA benchmarks\"\n \"The benchmarks should focus on complex, high-level reasoning that integrates gaze dynamics with event interactions.\"\n \"**Input data**:\"\n \"1. Visual: 9 first-person RGB frames + captions.\"\n \"2. Ego-centric gaze information: Includes gaze fixation points.\"\n \"**Event Parsing**:\"\n \"- Identify action chains (e.g., pick glass → drink → glance at bottle → pour water).\"\n \"- Link actions to gaze patterns, focusing on implicit cause-and-effect relationships.\"\n \"**Gaze Dynamics**:\" \n \"- Cluster gaze points as natural scene regions (e.g., 'cup rim', 'bottle cap area').\"\n \"- Track gaze trajectory shifts (e.g., 'suddenly locked onto...'). \"\n \"- Focus on: Gaze as a predictive signal for upcoming actions; Ambiguous gaze paths that may point to multiple plausible behaviors.\"\n \"**Requirements**:\"\n \"1. Ego-centric Question Construction: Create 5 multiple-choice options: 1 correct and 4 misleading.\"\n \"- Use templates like:\"\n \"Why did [Subject] perform [Action] while [Ongoing Task], given the observed changes in my attention?\"\n \"Why did [Subject] [Action] while [Other Subject] was [Action], considering the shifts in my attention?\"\n \"What was [Subject] trying to achieve by [Action], based on the changes in my attention during [Task]?\" \n \"2. Generate Answer Options Ensure that:\"\n \"- Correct answers must require concurrent gaze features to explain actions.\"\n \"- Avoid using external contextual clues that are not related to gaze.\" \n \"3. Spatio-Temporal Binding:\"\n \"- Express temporal relationships through event sequences, but avoid explicit time references (e.g., 'after Frame 3', 'during the first phase').\"\n \"- Focus on my gaze’s role in anticipating or influencing actions, but avoid overly simplistic or surface-level reasoning.\"\n \"4. Distractor Design:\"\n \"- Include reverse-causal options, where my gaze behavior is misinterpreted as being a result of the action.\"\n \"- Use spatial-proximity traps where objects in close proximity are incorrectly linked to my gaze behavior.\"\n \"- Introduce high-salience distractors that divert attention to irrelevant but visually prominent elements.\"\n \"- Create social influence traps by suggesting social behaviors or mimicry as the primary cause for the gaze behavior, even when it's not.\"\n \"5. Random Distribution of Correct Answer:\"\n \"- Ensure that the correct answer option is randomly distributed among the five options.\"\n \"Example:\"\n \"### Question:\\n\"\n \"Why did I shuffle the cards while organizing them, given the observed changes in attention?\"\n \"### Answer Options:\\n\"\n \"A: I was focusing on the deck to ensure it was properly shuffled.\"\n \"B: I was distracted by the cards' edges and kept adjusting them.\"\n \"C: I was checking if any specific card was missing from the deck.\"\n \"D: I was trying to hide certain cards from the others.\"\n \"E: I was preparing the cards for a trick.\"\n \"### Correct Answer:\\n\"\n \"C: I was checking if any specific card was missing from the deck.\"}\n ]\n },\n {\n \"role\": \"user\",\n \"content\":[\n {\"type\": \"video\",\n \"video\": image_data_list},\n {\"type\": \"text\",\n \"text\":f\"Given inputs: {joint_text}\\n\"\n \"Generate a causal reasoning, gaze-aware ego-centric video QA benchmark and give the correct answer.\"}\n ]\n }\n ]\n )\n print(completion.choices[0].message.content)\n completion = completion.choices[0].message.content\n # append_to_json(target_group_index, group_id, joint_text, completion, output_json)\n save_completion_to_csv(video_id, group_id, completion, output_csv)\n\n except Exception as e:\n print(f\"错误信息：{e}\")\n print(\"请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code\")\nif __name__ == \"__main__\":\n main()\n\n "}}}} |