1 line
10 KiB
JSON
1 line
10 KiB
JSON
{"repo": "ch-umar-aslam/multimodal-voice-agent", "files": {"app.py": {"3c7a1448b75fe9be63a5fdcba850b40b000ac8c0": {"t": 1785674068.4798644, "n": 9730, "h": "49bf2f7f7b4e563a", "c": "import streamlit as st\nimport os, time, asyncio, edge_tts\nfrom io import BytesIO\nfrom dotenv import load_dotenv\n\n# --- IMPORTS FOR BROWSER RECORDING ---\nfrom streamlit_mic_recorder import mic_recorder\nfrom pydub import AudioSegment # NEW: For converting WebM -> WAV\nfrom deepgram import DeepgramClient\nfrom langchain_community.vectorstores import FAISS\nfrom langchain_google_genai import GoogleGenerativeAIEmbeddings\nfrom langchain_openai import ChatOpenAI\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.output_parsers import StrOutputParser\nfrom langchain_core.runnables import RunnablePassthrough\n\n# --- CONFIGURATION ---\nst.set_page_config(layout=\"wide\", page_title=\"Virtuans Voice Agent\")\nload_dotenv()\n\n# Verify Keys\nif not os.getenv(\"OPENROUTER_API_KEY\") or not os.getenv(\"GOOGLE_API_KEY\"):\n st.error(\"🚨 Missing API Keys! Check your .env file.\")\n st.stop()\n\n# --- AUDIO INPUT\ndef speech_to_text(audio_bytes):\n \"\"\"\n Sends audio bytes directly to Deepgram using SDK v3.\n \"\"\"\n try:\n # Initialize Deepgram Client\n \n deepgram_key = os.getenv(\"DEEPGRAM_API_KEY\")\n deepgram = DeepgramClient(api_key=deepgram_key)\n\n # Configure Options\n \n options = {\n \"model\": \"nova-2\",\n \"smart_format\": True,\n \"language\": \"en-US\",\n }\n\n # Send Payload\n # Deepgram expects a dictionary payload for buffers\n payload = {\"buffer\": audio_bytes}\n \n # Transcribe\n response = deepgram.listen.prerecorded.v(\"1\").transcribe_file(payload, options)\n \n # Extract Transcript\n transcript = response.results.channels[0].alternatives[0].transcript\n return transcript\n\n except Exception as e:\n st.error(f\"Deepgram STT Error: {e}\")\n return None\n\n# --- AUDIO OUTPUT UTILS (EdgeTTS) ---\ndef text_to_speech(text, model_name):\n \"\"\"Converts text to audio using Edge-TTS.\"\"\"\n if not text:\n return None\n \n # Assign specific voices\n voice = \"en-US-AriaNeural\"\n if \"Gemini\" in model_name:\n voice = \"en-US-AvaNeural\"\n elif \"DeepSeek\" in model_name:\n voice = \"en-US-ChristopherNeural\"\n elif \"Kimi\" in model_name:\n voice = \"en-US-EricNeural\"\n\n output_file = f\"temp_{model_name}.mp3\"\n \n async def generate():\n communicate = edge_tts.Communicate(text, voice)\n await communicate.save(output_file)\n\n try:\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n loop.run_until_complete(generate())\n loop.close()\n \n with open(output_file, \"rb\") as f:\n audio_bytes = BytesIO(f.read())\n \n if os.path.exists(output_file):\n os.remove(output_file)\n \n return audio_bytes\n \n except Exception as e:\n print(f\"TTS Error for {model_name}: {e}\")\n return None\n\n# --- RAG PIPELINE ---\n@st.cache_resource\ndef setup_rag_chain():\n embeddings = GoogleGenerativeAIEmbeddings(model=\"models/text-embedding-004\", max_retries=5)\n \n try:\n vectorstore = FAISS.load_local(\"faiss_index\", embeddings, allow_dangerous_deserialization=True)\n retriever = vectorstore.as_retriever(search_kwargs={\"k\": 3})\n except Exception as e:\n st.error(f\"Error loading Vector DB: {e}. Run ingest.py first!\")\n return None\n\n template = \"\"\"You are a specialized AI Assistant for Sunmarke School.\n\n TASK:\n 1. If the answer is in the context, provide a concise and short response (max 3 sentences).\n 2. If the question is NOT about Sunmarke School or the information is missing, you must politely shortly introduce your specialization.\n\n DYNAMIC FALLBACK RULE:\n - Do NOT use a fixed or hardcoded response for out-of-scope questions.\n - Instead, in your own unique personality and style, precise that you are a dedicated Sunmarke School assistant and can only help with school-related matters (like admissions, fees, or curriculum).\n - Be creative but professional in how you phrase your limitation. \n - Mention the user's topic to show you understood it, then pivot back to your role at Sunmarke.\n\n Context: {context}\n Question: {question}\n\n Response:\"\"\"\n prompt = ChatPromptTemplate.from_template(template)\n\n # 3. Initialize Models (Fixed IDs + Max Tokens)\n openrouter_base = \"https://openrouter.ai/api/v1\"\n openrouter_key = os.getenv(\"OPENROUTER_API_KEY\")\n headers = {\"HTTP-Referer\": \"http://localhost:8501\", \"X-Title\": \"Virtuans Agent\"}\n\n # --- UPDATED MODEL IDs HERE ---\n llm_gemini = ChatOpenAI(\n api_key=openrouter_key, \n model=\"google/gemini-2.5-flash\",\n base_url=openrouter_base, \n temperature=0,\n default_headers=headers,\n max_tokens=500\n \n )\n \n llm_deepseek = ChatOpenAI(\n model=\"deepseek/deepseek-chat\",\n api_key=openrouter_key, \n base_url=openrouter_base, \n temperature=0,\n default_headers=headers,\n max_tokens=500\n )\n \n llm_kimi = ChatOpenAI(\n model=\"moonshotai/kimi-k2-0905\", \n api_key=openrouter_key, \n base_url=openrouter_base,\n temperature=0, \n default_headers=headers,\n max_tokens=500\n )\n\n rag_setup = {\"context\": retriever | (lambda d: \"\\n\\n\".join([x.page_content for x in d])), \"question\": RunnablePassthrough()}\n\n return {\n \"gemini\": rag_setup | prompt | llm_gemini | StrOutputParser(),\n \"deepseek\": rag_setup | prompt | llm_deepseek | StrOutputParser(),\n \"kimi\": rag_setup | prompt | llm_kimi | StrOutputParser()\n }\n\n# --- MAIN UI ---\ndef main():\n st.title(\"🤖 Virtuans Voice Agent\")\n \n chains = setup_rag_chain()\n if not chains: st.stop()\n\n col_input, col_text = st.columns([1, 4])\n user_query = None\n\n # --- INPUT: TEXT FALLBACK ---\n with col_text:\n if \"text_input\" not in st.session_state:\n st.session_state[\"text_input\"] = \"\"\n text_input = st.text_input(\n \"Or type here:\",\n value=st.session_state[\"text_input\"],\n key=\"text_input_widget\"\n )\n if text_input:\n user_query = text_input\n \n # --- INPUT: VOICE ---\n with col_input:\n st.write(\"Click to Speak:\")\n audio = mic_recorder(\n start_prompt=\"🎙️ Voice\",\n stop_prompt=\"⏹️ Stop\",\n key='recorder',\n just_once=False,\n use_container_width=True\n )\n \n if audio:\n with st.spinner(\"Transcribing...\"):\n user_query = speech_to_text(audio['bytes'])\n # Clear text input after voice transcription\n st.session_state[\"text_input\"] = \"\"\n\n elif text_input:\n user_query = text_input\n # Clear text input after submission\n st.session_state[\"text_input\"] = \"\"\n\n\n \n\n # --- PROCESSING ---\n if user_query:\n start_time = time.perf_counter()\n st.divider()\n \n st.subheader(f\"Q: {user_query}\")\n \n col1, col2, col3 = st.columns(3)\n with col1: status_g = st.empty(); status_g.info(\"Gemini thinking...\")\n with col2: status_k = st.empty(); status_k.info(\"Kimi thinking...\")\n with col3: status_d = st.empty(); status_d.info(\"DeepSeek thinking...\")\n\n async def get_model_response(name, chain, q):\n start = time.perf_counter()\n try:\n response = await chain.ainvoke(q)\n duration = time.perf_counter() - start\n return response, duration\n except Exception as e:\n return e, 0\n\n async def get_responses(q):\n return await asyncio.gather(\n get_model_response(\"gemini\", chains[\"gemini\"], q),\n get_model_response(\"kimi\", chains[\"kimi\"], q),\n get_model_response(\"deepseek\", chains[\"deepseek\"], q),\n return_exceptions=True\n )\n\n try:\n results = asyncio.run(get_responses(user_query))\n total_duration = time.perf_counter() - start_time\n resp_g, resp_k, resp_d = results\n \n for s in [status_g, status_k, status_d]: s.empty()\n\n def display_result(col, name, resp_data):\n \n resp, duration = resp_data\n\n with col:\n st.markdown(f\"### {name}\")\n if isinstance(resp, Exception):\n st.error(f\"API Error: {str(resp)}\")\n else:\n st.success(resp)\n \n \n \n c1, c2 ,c3= st.columns(3)\n with c1: st.caption(\"👍\")\n with c2: st.caption(\"👎\")\n with c3: st.caption(f\"⏱ {duration:.2f}s\")\n\n \n # Audio output\n audio_fp = text_to_speech(resp, name)\n if audio_fp: \n st.audio(audio_fp, format=\"audio/mp3\")\n else:\n st.warning(\"Audio unavailable\")\n\n # Calling part (In main loop)\n results = asyncio.run(get_responses(user_query))\n resp_g_data, resp_k_data, resp_d_data = results\n\n display_result(col1, \"Gemini\", resp_g_data)\n display_result(col2, \"Kimi\", resp_k_data)\n display_result(col3, \"DeepSeek\", resp_d_data)\n \n\n except Exception as e:\n st.error(f\"System Error: {e}\")\n\nif __name__ == \"__main__\":\n main()"}}}} |