{"repo": "anonyupload/egogazevqa", "files": {"generate_tool/causal.py": {"a9f1c5d5e006988f5a6b57d4a53d9a3716a562b8": {"t": 1785846856.902, "n": 13500, "h": "1bc33913516e5968", "c": "import os\nimport json\nimport csv\nfrom openai import OpenAI\nimport base64\nimport argparse\n\n\ndef group_frames_and_generate_text(image_folder, json_file,  target_group_index, group_size=9):\n    \"\"\"\n    Groups images from a folder, retrieves related narration and gaze info from JSON,\n    and generates text pairs in the format \"Frame X [Caption; gaze info]\".\n    \n    :param image_folder: Path to the folder containing image files.\n    :param json_file: Path to the JSON file containing narration data.\n    :param group_size: Number of images per group (default: 9).\n    \"\"\"\n    try:\n        # Load narration JSON\n        with open(json_file, \"r\") as f:\n            narrations = json.load(f)\n\n        # Retrieve the video ID from the folder name\n        video_id = os.path.basename(image_folder)\n        print(f\"Video_id: {video_id}\\nCurrent group:{target_group_index}\")\n        \n        # Ensure the JSON contains data for this video ID\n        if video_id not in narrations:\n            raise ValueError(f\"Video ID {video_id} not found in the JSON file.\")\n        \n        narration_data = narrations[video_id][\"narration_pass_1\"][\"narrations\"]\n\n        # Create a mapping of frame numbers to their narration and gaze info\n        frame_to_narration = {\n            item[\"timestamp_frame\"]: {\n                \"narration_text\": item[\"narration_text\"],\n                \"gaze_info\": item.get(\"gaze_info\", {})\n            }\n            for item in narration_data\n        }\n\n        # List and sort image files\n        image_files = sorted(\n            [f for f in os.listdir(image_folder) if f.endswith(('.jpg'))],\n            key=lambda x: int(os.path.splitext(x)[0])\n        )\n\n        # Group images and generate text pairs\n        text_pairs = []\n        for i in range(0, len(image_files), group_size):\n            group_index = i // group_size  # 当前分组的索引\n            if group_index == target_group_index:  # 只处理目标分组\n                group = image_files[i:i + group_size]\n                for j, image_file in enumerate(group):\n                    frame_number = int(os.path.splitext(image_file)[0])  # 从文件名提取帧号\n                    if frame_number in frame_to_narration:\n                        narration_text = frame_to_narration[frame_number][\"narration_text\"]\n                        gaze_info = frame_to_narration[frame_number][\"gaze_info\"]\n                        gaze_text = f\"Gaze:({gaze_info.get('gaze_x', 0)},{gaze_info.get('gaze_y', 0)})\"\n                        text_pair = f\"Frame {j + 1}: {narration_text}; {gaze_text}\"\n                        text_pairs.append(text_pair)\n                    else:\n                        print(f\"No narration data for frame {frame_number}, skipping.\")\n        \n        return text_pairs, group\n    \n    except Exception as e:\n        print(f\"Error: {e}\")\n    \n\ndef encode_images_from_folder(folder_path, group_id):\n    \"\"\"\n    Encode all image files in a folder to Base64 and return them as a list of dictionaries.\n    \n    :param folder_path: Path to the folder containing images.\n    :return: List of dictionaries with Base64 encoded images in the required format.\n    \"\"\"\n    image_data_list = []\n    # Sort files to ensure correct order\n    \n    for image_file in group_id:\n        image_path = os.path.join(folder_path, image_file)\n        with open(image_path, \"rb\") as img_file:\n            base64_image = base64.b64encode(img_file.read()).decode(\"utf-8\")\n            image_data_list.append(f\"data:image/jpeg;base64,{base64_image}\")\n    return image_data_list\n\nimport json\n\ndef append_to_json(target_group_index, group_id, caption, completion_content, output_path):\n    \"\"\"\n    Append new data to an existing JSON file or create a new one.\n\n    :param video_id: ID of the video being processed.\n    :param target_group_index: Index of the current target group.\n    :param group_id: ID of the group being processed.\n    :param completion_content: Content generated by the completion model.\n    :param output_path: Path to save the JSON file.\n    \"\"\"\n\n    new_entry = {\n        \"current_group\": target_group_index,\n        \"group_id\": group_id,\n        \"caption\": caption,\n        \"completion_content\": completion_content\n    }\n\n\n    if os.path.exists(output_path):\n        with open(output_path, \"r\") as json_file:\n            existing_data = json.load(json_file)\n    else:\n        existing_data = []\n\n\n    existing_data.append(new_entry)\n\n\n    with open(output_path, \"w\") as json_file:\n        json.dump(existing_data, json_file, indent=4)\n\n    print(f\"New entry appended to {output_path}\")\n\ndef save_completion_to_csv(video_id, group_id, completion_content, output_csv):\n    \"\"\"\n    Extract information from completion content and save to CSV file.\n\n    :param video_id: ID of the video.\n    :param group_id: ID of the group.\n    :param completion_content: Completion content to parse.\n    :param output_csv: Path to the output CSV file.\n    \"\"\"\n \n    lines = completion_content.split(\"\\n\")\n    question = \"\"\n    answer_options = []\n    correct_answer = \"\"\n    \n\n    is_answer_section = False\n\n    for i, line in enumerate(lines):\n        line = line.strip()\n        if line.startswith(\"### Question\"):\n     \n            for j in range(i + 1, len(lines)):\n                next_line = lines[j].strip()\n                if next_line.startswith(\"### Answer Options:\"):  \n                    break\n                if \"what\" in next_line.lower() or \"where\" in next_line.lower() or \"when\" in next_line.lower() or \"why\" in next_line.lower() or \"which\" in next_line.lower() or \"how\" in next_line.lower():  # 查找包含 \"question\" 的行\n                    question = next_line.strip()\n        elif line.startswith(\"### Answer Options:\"):\n            is_answer_section = True  \n        elif line.startswith(\"### Correct Answer:\"):\n            is_answer_section = False  \n            correct_answer = lines[i + 1][0].strip()  \n        elif is_answer_section and (line.startswith(\"A:\") or line.startswith(\"B:\") or line.startswith(\"C:\") or line.startswith(\"D:\") or line.startswith(\"E:\")):\n            answer_options.append(line)  \n    \n    file_exists = os.path.exists(output_csv)\n   \n    data = [\n        {\n            \"video_id\": video_id,\n            \"group_id\": \"\\n\".join(group_id),\n            \"Question\": question,\n            \"Answer Options\": \"\\n\".join(answer_options),\n            \"Correct Answer\": correct_answer\n        }\n    ]\n\n\n    with open(output_csv, \"a\", newline=\"\", encoding=\"utf-8\") as csvfile:\n        fieldnames = [\"video_id\", \"group_id\", \"Question\", \"Answer Options\", \"Correct Answer\"]\n        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n\n     \n        if not file_exists:\n            writer.writeheader()\n\n        writer.writerows(data)\n    \n    print(f\"Data appended to {output_csv}\")\n\ndef main():\n    parser = argparse.ArgumentParser(description=\" video_id and target_group_index \")\n\n    parser.add_argument('--video_id', type=str, required=True)\n    parser.add_argument('--target_index', type=int, required=True)\n\n    args = parser.parse_args()\n\n    video_id = args.video_id\n    target_group_index = args.target_index\n\n  \n    image_folder = f\"/home/pty_ssd/output0207/{video_id}\"\n    json_file = \"/home/pty_ssd/narration_with_gaze_600s.json\"\n    output_json = f\"/home/pty_ssd/Qwen2-VL/QA_benchmark0207/reasoning/json/{video_id}.json\"\n    output_csv = f\"/home/pty_ssd/Qwen2-VL/QA_benchmark0207/reasoning/csv_v3/{video_id}.csv\"\n\n\n    text_pairs, group_id = group_frames_and_generate_text(image_folder, json_file, target_group_index)\n    print(f\"Group_id: {group_id}\")\n    image_data_list = encode_images_from_folder(image_folder, group_id)\n\n    joint_text = \"\\n\".join(text_pairs)\n\n    try:\n        client = OpenAI(\n            api_key='sk-',\n            base_url=\"https://dashscope.aliyuncs.com/compatible-mode/v1\",\n        )\n\n\n        completion = client.chat.completions.create(\n        model=\"qwen-vl-max-latest\",\n        messages=[\n            {\n                \"role\": \"system\",\n                \"content\":[\n                    {\"type\": \"text\",\n                        \"text\": \"You are an expert in gaze-based event causal understanding within ego-centric video environments. Your task is to design comprehensive Gaze-Informed Causal Reasoning ego-centric VideoQA benchmarks\"\n                                \"The benchmarks should focus on complex, high-level reasoning that integrates gaze dynamics with event interactions.\"\n                                \"**Input data**:\"\n                                \"1. Visual: 9 first-person RGB frames + captions.\"\n                                \"2. Ego-centric gaze information: Includes gaze fixation points.\"\n                                \"**Event Parsing**:\"\n                                    \"- Identify action chains (e.g., pick glass → drink → glance at bottle → pour water).\"\n                                    \"- Link actions to gaze patterns, focusing on implicit cause-and-effect relationships.\"\n                                \"**Gaze Dynamics**:\"  \n                                    \"- Cluster gaze points as natural scene regions (e.g., 'cup rim', 'bottle cap area').\"\n                                    \"- Track gaze trajectory shifts (e.g., 'suddenly locked onto...'). \"\n                                    \"- Focus on: Gaze as a predictive signal for upcoming actions; Ambiguous gaze paths that may point to multiple plausible behaviors.\"\n                                \"**Requirements**:\"\n                                \"1. Ego-centric Question Construction: Create 5 multiple-choice options: 1 correct and 4 misleading.\"\n                                    \"- Use templates like:\"\n                                    \"Why did [Subject] perform [Action] while [Ongoing Task], given the observed changes in my attention?\"\n                                    \"Why did [Subject] [Action] while [Other Subject] was [Action], considering the shifts in my attention?\"\n                                    \"What was [Subject] trying to achieve by [Action], based on the changes in my attention during [Task]?\"                            \n                                \"2. Generate Answer Options Ensure that:\"\n                                    \"- Correct answers must require concurrent gaze features to explain actions.\"\n                                    \"- Avoid using external contextual clues that are not related to gaze.\" \n                                \"3. Spatio-Temporal Binding:\"\n                                    \"- Express temporal relationships through event sequences, but avoid explicit time references (e.g., 'after Frame 3', 'during the first phase').\"\n                                    \"- Focus on my gaze’s role in anticipating or influencing actions, but avoid overly simplistic or surface-level reasoning.\"\n                                \"4. Distractor Design:\"\n                                    \"- Include reverse-causal options, where my gaze behavior is misinterpreted as being a result of the action.\"\n                                    \"- Use spatial-proximity traps where objects in close proximity are incorrectly linked to my gaze behavior.\"\n                                    \"- Introduce high-salience distractors that divert attention to irrelevant but visually prominent elements.\"\n                                    \"- Create social influence traps by suggesting social behaviors or mimicry as the primary cause for the gaze behavior, even when it's not.\"\n                                \"5. Random Distribution of Correct Answer:\"\n                                    \"- Ensure that the correct answer option is randomly distributed among the five options.\"\n                                \"Example:\"\n                                \"### Question:\\n\"\n                                \"Why did I shuffle the cards while organizing them, given the observed changes in attention?\"\n                                \"### Answer Options:\\n\"\n                                \"A: I was focusing on the deck to ensure it was properly shuffled.\"\n                                \"B: I was distracted by the cards' edges and kept adjusting them.\"\n                                \"C: I was checking if any specific card was missing from the deck.\"\n                                \"D: I was trying to hide certain cards from the others.\"\n                                \"E: I was preparing the cards for a trick.\"\n                                \"### Correct Answer:\\n\"\n                                \"C: I was checking if any specific card was missing from the deck.\"}\n                ]\n            },\n            {\n                \"role\": \"user\",\n                \"content\":[\n                    {\"type\": \"video\",\n                        \"video\": image_data_list},\n                    {\"type\": \"text\",\n                            \"text\":f\"Given inputs: {joint_text}\\n\"\n                                    \"Generate a causal reasoning, gaze-aware ego-centric video QA benchmark and give the correct answer.\"}\n                ]\n            }\n        ]\n    )\n        print(completion.choices[0].message.content)\n        completion = completion.choices[0].message.content\n        # append_to_json(target_group_index, group_id, joint_text, completion, output_json)\n        save_completion_to_csv(video_id, group_id, completion, output_csv)\n\n    except Exception as e:\n        print(f\"错误信息：{e}\")\n        print(\"请参考文档：https://help.aliyun.com/zh/model-studio/developer-reference/error-code\")\nif __name__ == \"__main__\":\n    main()\n\n    "}}}}