1 line
20 KiB
JSON
1 line
20 KiB
JSON
{"repo": "amine-sabbahi/omni-ai-web-scraper-app", "files": {"Back-end/rag_scraper/configs.py": {"9cd02ea9bcce2c0e03ee5b78ea38003dd4e093e3": {"t": 1785745459.458339, "n": 3782, "h": "940caca350e7c226", "c": "# Back-end/rag_scraper/configs.py\n\n\"\"\"\nThis module contains configuration variables and constants\nthat are used across different parts of the application.\n\"\"\"\n\n\nSUPABASE_MARKDOWNS_TABLE = \"markdowns_table\"\nSUPABASE_EMBEDDINGS_TABLE = \"embeddings_table\"\n\n\n\nGEMINI_MODEL_FULLNAME=\"gemini/gemini-1.5-flash\"\nOPENAI_MODEL_FULLNAME=\"gpt-4o-mini\"\nDEEPSEEK_MODEL_FULLNAME =\"groq/deepseek-r1-distill-llama-70b\"\nLLAMA3_MODEL_FULLNAME =\"groq/llama-3.3-70b-versatile\"\nMISTRAL_MODEL_FULLNAME=\"mistral/mistral-large-latest\"\nMODELS_USED = {\n OPENAI_MODEL_FULLNAME: {\"OPENAI_API_KEY\"},\n GEMINI_MODEL_FULLNAME: {\"GEMINI_API_KEY\"},\n DEEPSEEK_MODEL_FULLNAME : {\"GROQ_DEEPSEEK_API_KEY\"},\n LLAMA3_MODEL_FULLNAME : {\"GROQ_LLAMA_API_KEY\"},\n MISTRAL_MODEL_FULLNAME : {\"MISTRAL_API_KEY\"},\n}\n\n\n\n# Timeout settings for web scraping\nTIMEOUT_SETTINGS = {\n \"page_load\": 30,\n \"script\": 10\n}\n\nNUMBER_SCROLL=2\n\n# RAG SYSTEM INSTRUCTIONS \n\nRAG_USER_MESSAGE = f\"Extract the following information from the provided text:\\n\\n\"\n \n\nRAG_SYSTEM_MESSAGE = (\n \"Answer the question based solely on the provided context. \"\n \"If insufficient context is provided, mention that information is missing.\"\n \"If the user is saying hello, or just talking to you without any context or question, \"\n \"you can respond with a greeting or a general response.\"\n )\n\nEMBEDDING_COST_PER_1M_TOKENS = 0 # $0.0 per 1M tokens for text-embedding-004 (google embedding model)\n\n# SCRAPER SYSTEM INSTRUCTIONS\nSYSTEM_MESSAGE = \"\"\"You are an intelligent text extraction and conversion assistant. Your task is to extract structured information \n from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, \n with no additional commentary, explanations, or extraneous information. \n You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.\n Please process the following text and provide the output in pure JSON format with no words before or after the JSON:\"\"\"\n\nUSER_MESSAGE = f\"Extract the following information from the provided text:\\nPage content:\\n\\n\"\n \n\n\n\n\nPROMPT_PAGINATION = \"\"\"\nYou are an assistant that extracts pagination URLs from markdown content of websites. \nYour task is to identify and generate a list of pagination URLs based on a detected URL pattern where page numbers increment sequentially. Follow these instructions carefully:\n\n-Identify the Pagination Pattern:\nAnalyze the provided markdown text to detect URLs that follow a pattern where only a numeric page indicator changes.\nIf the numbers start from a low value and increment, generate the full sequence of URLs—even if not all numbers are present in the text.\n\n-Construct Complete URLs:\nIn cases where only part of a URL is provided, combine it with the given base URL (which will appear at the end of this prompt) to form complete URLs.\nEnsure that every URL you generate is clickable and leads directly to the intended page.\n\n-Incorporate User Indications:\nIf additional user instructions about the pagination mechanism are provided at the end of the prompt, use those instructions to refine your URL generation.\nOutput Format Requirements:\n\n-Strictly output only a valid JSON object with the exact structure below:\n\"\"\n{\n \"page_urls\": [\"url1\", \"url2\", \"url3\", ..., \"urlN\"]\n}\"\"\n\n\nIMPORTANT:\n\nOutput only a single valid JSON object with no additional text, markdown formatting, or explanation.\nDo not include any extra newlines or spaces before or after the JSON.\nThe JSON object must exactly match the following schema:\n\"\"\"\n"}, "b77b9b724e936d4255be05b00a96557ec386f6da": {"t": 1785753498.8589447, "n": 3782, "h": "940caca350e7c226", "c": "# Back-end/rag_scraper/configs.py\n\n\"\"\"\nThis module contains configuration variables and constants\nthat are used across different parts of the application.\n\"\"\"\n\n\nSUPABASE_MARKDOWNS_TABLE = \"markdowns_table\"\nSUPABASE_EMBEDDINGS_TABLE = \"embeddings_table\"\n\n\n\nGEMINI_MODEL_FULLNAME=\"gemini/gemini-1.5-flash\"\nOPENAI_MODEL_FULLNAME=\"gpt-4o-mini\"\nDEEPSEEK_MODEL_FULLNAME =\"groq/deepseek-r1-distill-llama-70b\"\nLLAMA3_MODEL_FULLNAME =\"groq/llama-3.3-70b-versatile\"\nMISTRAL_MODEL_FULLNAME=\"mistral/mistral-large-latest\"\nMODELS_USED = {\n OPENAI_MODEL_FULLNAME: {\"OPENAI_API_KEY\"},\n GEMINI_MODEL_FULLNAME: {\"GEMINI_API_KEY\"},\n DEEPSEEK_MODEL_FULLNAME : {\"GROQ_DEEPSEEK_API_KEY\"},\n LLAMA3_MODEL_FULLNAME : {\"GROQ_LLAMA_API_KEY\"},\n MISTRAL_MODEL_FULLNAME : {\"MISTRAL_API_KEY\"},\n}\n\n\n\n# Timeout settings for web scraping\nTIMEOUT_SETTINGS = {\n \"page_load\": 30,\n \"script\": 10\n}\n\nNUMBER_SCROLL=2\n\n# RAG SYSTEM INSTRUCTIONS \n\nRAG_USER_MESSAGE = f\"Extract the following information from the provided text:\\n\\n\"\n \n\nRAG_SYSTEM_MESSAGE = (\n \"Answer the question based solely on the provided context. \"\n \"If insufficient context is provided, mention that information is missing.\"\n \"If the user is saying hello, or just talking to you without any context or question, \"\n \"you can respond with a greeting or a general response.\"\n )\n\nEMBEDDING_COST_PER_1M_TOKENS = 0 # $0.0 per 1M tokens for text-embedding-004 (google embedding model)\n\n# SCRAPER SYSTEM INSTRUCTIONS\nSYSTEM_MESSAGE = \"\"\"You are an intelligent text extraction and conversion assistant. Your task is to extract structured information \n from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, \n with no additional commentary, explanations, or extraneous information. \n You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.\n Please process the following text and provide the output in pure JSON format with no words before or after the JSON:\"\"\"\n\nUSER_MESSAGE = f\"Extract the following information from the provided text:\\nPage content:\\n\\n\"\n \n\n\n\n\nPROMPT_PAGINATION = \"\"\"\nYou are an assistant that extracts pagination URLs from markdown content of websites. \nYour task is to identify and generate a list of pagination URLs based on a detected URL pattern where page numbers increment sequentially. Follow these instructions carefully:\n\n-Identify the Pagination Pattern:\nAnalyze the provided markdown text to detect URLs that follow a pattern where only a numeric page indicator changes.\nIf the numbers start from a low value and increment, generate the full sequence of URLs—even if not all numbers are present in the text.\n\n-Construct Complete URLs:\nIn cases where only part of a URL is provided, combine it with the given base URL (which will appear at the end of this prompt) to form complete URLs.\nEnsure that every URL you generate is clickable and leads directly to the intended page.\n\n-Incorporate User Indications:\nIf additional user instructions about the pagination mechanism are provided at the end of the prompt, use those instructions to refine your URL generation.\nOutput Format Requirements:\n\n-Strictly output only a valid JSON object with the exact structure below:\n\"\"\n{\n \"page_urls\": [\"url1\", \"url2\", \"url3\", ..., \"urlN\"]\n}\"\"\n\n\nIMPORTANT:\n\nOutput only a single valid JSON object with no additional text, markdown formatting, or explanation.\nDo not include any extra newlines or spaces before or after the JSON.\nThe JSON object must exactly match the following schema:\n\"\"\"\n"}, "882af4b936e2f6c95bf42928bc0930d562c5dea9": {"t": 1785753500.9326887, "n": 3783, "h": "9742b63b6a45745f", "c": "# Back-end/rag_scraper/configs.py\n\n\"\"\"\nThis module contains configuration variables and constants\nthat are used across different parts of the application.\n\"\"\"\n\n\nSUPABASE_MARKDOWNS_TABLE = \"markdowns_table\"\nSUPABASE_EMBEDDINGS_TABLE = \"embeddings_table\"\n\n\n\nGEMINI_MODEL_FULLNAME=\"gemini/gemini-1.5-flash\"\nOPENAI_MODEL_FULLNAME=\"gpt-4o-mini\"\nDEEPSEEK_MODEL_FULLNAME =\"groq/deepseek-r1-distill-llama-70b\"\nLLAMA3_MODEL_FULLNAME =\"groq/llama-3.3-70b-versatile\"\nMISTRAL_MODEL_FULLNAME=\"mistral/mistral-large-latest\"\nMODELS_USED = {\n OPENAI_MODEL_FULLNAME: {\"OPENAI_API_KEY\"},\n GEMINI_MODEL_FULLNAME: {\"GEMINI_API_KEY\"},\n DEEPSEEK_MODEL_FULLNAME : {\"GROQ_DEEPSEEK_API_KEY\"},\n LLAMA3_MODEL_FULLNAME : {\"GROQ_LLAMA_API_KEY\"},\n MISTRAL_MODEL_FULLNAME : {\"MISTRAL_API_KEY\"},\n}\n\n\n\n# Timeout settings for web scraping\nTIMEOUT_SETTINGS = {\n \"page_load\": 30,\n \"script\": 10\n}\n\nNUMBER_SCROLL=2\n\n# RAG SYSTEM INSTRUCTIONS \n\nRAG_USER_MESSAGE = f\"Extract the following information from the provided text:\\n\\n\"\n \n\nRAG_SYSTEM_MESSAGE = (\n \"Answer the question based solely on the provided context. \"\n \"If insufficient context is provided, mention that information is missing.\"\n \"If the user is saying hello, or just talking to you without any context or question, \"\n \"you can respond with a greeting or a general response.\"\n )\n\nEMBEDDING_COST_PER_1M_TOKENS = 0 # $0.10 per 1M tokens for text-embedding-004 (google embedding model)\n\n# SCRAPER SYSTEM INSTRUCTIONS\nSYSTEM_MESSAGE = \"\"\"You are an intelligent text extraction and conversion assistant. Your task is to extract structured information \n from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, \n with no additional commentary, explanations, or extraneous information. \n You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.\n Please process the following text and provide the output in pure JSON format with no words before or after the JSON:\"\"\"\n\nUSER_MESSAGE = f\"Extract the following information from the provided text:\\nPage content:\\n\\n\"\n \n\n\n\n\nPROMPT_PAGINATION = \"\"\"\nYou are an assistant that extracts pagination URLs from markdown content of websites. \nYour task is to identify and generate a list of pagination URLs based on a detected URL pattern where page numbers increment sequentially. Follow these instructions carefully:\n\n-Identify the Pagination Pattern:\nAnalyze the provided markdown text to detect URLs that follow a pattern where only a numeric page indicator changes.\nIf the numbers start from a low value and increment, generate the full sequence of URLs—even if not all numbers are present in the text.\n\n-Construct Complete URLs:\nIn cases where only part of a URL is provided, combine it with the given base URL (which will appear at the end of this prompt) to form complete URLs.\nEnsure that every URL you generate is clickable and leads directly to the intended page.\n\n-Incorporate User Indications:\nIf additional user instructions about the pagination mechanism are provided at the end of the prompt, use those instructions to refine your URL generation.\nOutput Format Requirements:\n\n-Strictly output only a valid JSON object with the exact structure below:\n\"\"\n{\n \"page_urls\": [\"url1\", \"url2\", \"url3\", ..., \"urlN\"]\n}\"\"\n\n\nIMPORTANT:\n\nOutput only a single valid JSON object with no additional text, markdown formatting, or explanation.\nDo not include any extra newlines or spaces before or after the JSON.\nThe JSON object must exactly match the following schema:\n\"\"\"\n"}, "76df8a57a0a1a5dccc719f398f444bf5dfd7d004": {"t": 1785753502.4718375, "n": 3760, "h": "7924bef05f51aa97", "c": "\"\"\"\nThis module contains configuration variables and constants\nthat are used across different parts of the application.\n\"\"\"\n\n# assets.py\n\nSUPABASE_MARKDOWNS_TABLE = \"markdowns_table\"\nSUPABASE_EMBEDDINGS_TABLE = \"embeddings_table\"\n\n\n\nGEMINI_MODEL_FULLNAME=\"gemini/gemini-1.5-flash\"\nOPENAI_MODEL_FULLNAME=\"gpt-4o-mini\"\nDEEPSEEK_MODEL_FULLNAME =\"groq/deepseek-r1-distill-llama-70b\"\nLLAMA3_MODEL_FULLNAME =\"groq/llama-3.3-70b-versatile\"\nMISTRAL_MODEL_FULLNAME=\"mistral/mistral-large-latest\"\nMODELS_USED = {\n OPENAI_MODEL_FULLNAME: {\"OPENAI_API_KEY\"},\n GEMINI_MODEL_FULLNAME: {\"GEMINI_API_KEY\"},\n DEEPSEEK_MODEL_FULLNAME : {\"GROQ_DEEPSEEK_API_KEY\"},\n LLAMA3_MODEL_FULLNAME : {\"GROQ_LLAMA_API_KEY\"},\n MISTRAL_MODEL_FULLNAME : {\"MISTRAL_API_KEY\"},\n}\n\n\n\n# Timeout settings for web scraping\nTIMEOUT_SETTINGS = {\n \"page_load\": 30,\n \"script\": 10\n}\n\nNUMBER_SCROLL=2\n\n# RAG SYSTEM INSTRUCTIONS \n\nRAG_USER_MESSAGE = f\"Extract the following information from the provided text:\\n\\n\"\n \n\nRAG_SYSTEM_MESSAGE = (\n \"Answer the question based solely on the provided context. \"\n \"If insufficient context is provided, mention that information is missing.\"\n \"If the user is saying hello, or just talking to you without any context or question, \"\n \"you can respond with a greeting or a general response.\"\n )\n\nEMBEDDING_COST_PER_1M_TOKENS = 0 # $0.10 per 1M tokens for text-embedding-004 (google embedding model)\n\n# SCRAPER SYSTEM INSTRUCTIONS\nSYSTEM_MESSAGE = \"\"\"You are an intelligent text extraction and conversion assistant. Your task is to extract structured information \n from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, \n with no additional commentary, explanations, or extraneous information. \n You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.\n Please process the following text and provide the output in pure JSON format with no words before or after the JSON:\"\"\"\n\nUSER_MESSAGE = f\"Extract the following information from the provided text:\\nPage content:\\n\\n\"\n \n\n\n\n\nPROMPT_PAGINATION = \"\"\"\nYou are an assistant that extracts pagination URLs from markdown content of websites. \nYour task is to identify and generate a list of pagination URLs based on a detected URL pattern where page numbers increment sequentially. Follow these instructions carefully:\n\n-Identify the Pagination Pattern:\nAnalyze the provided markdown text to detect URLs that follow a pattern where only a numeric page indicator changes.\nIf the numbers start from a low value and increment, generate the full sequence of URLs—even if not all numbers are present in the text.\n\n-Construct Complete URLs:\nIn cases where only part of a URL is provided, combine it with the given base URL (which will appear at the end of this prompt) to form complete URLs.\nEnsure that every URL you generate is clickable and leads directly to the intended page.\n\n-Incorporate User Indications:\nIf additional user instructions about the pagination mechanism are provided at the end of the prompt, use those instructions to refine your URL generation.\nOutput Format Requirements:\n\n-Strictly output only a valid JSON object with the exact structure below:\n\"\"\n{\n \"page_urls\": [\"url1\", \"url2\", \"url3\", ..., \"urlN\"]\n}\"\"\n\n\nIMPORTANT:\n\nOutput only a single valid JSON object with no additional text, markdown formatting, or explanation.\nDo not include any extra newlines or spaces before or after the JSON.\nThe JSON object must exactly match the following schema:\n\"\"\"\n"}, "19f1201e978e3465681728b9bd9fcc3432cb645d": {"t": 1785753503.6469417, "n": 3760, "h": "e6840a8184a2fe34", "c": "\"\"\"\nThis module contains configuration variables and constants\nthat are used across different parts of the application.\n\"\"\"\n\n# assets.py\n\nSUPABASE_MARKDOWNS_TABLE = \"markdowns_table\"\nSUPABASE_EMBEDDINGS_TABLE = \"embeddings_table\"\n\n\n\nGEMINI_MODEL_FULLNAME=\"gemini/gemini-1.5-flash\"\nOPENAI_MODEL_FULLNAME=\"gpt-4o-mini\"\nDEEPSEEK_MODEL_FULLNAME =\"groq/deepseek-r1-distill-llama-70b\"\nLLAMA3_MODEL_FULLNAME =\"groq/llama-3.1-70b-versatile\"\nMISTRAL_MODEL_FULLNAME=\"mistral/mistral-large-latest\"\nMODELS_USED = {\n OPENAI_MODEL_FULLNAME: {\"OPENAI_API_KEY\"},\n GEMINI_MODEL_FULLNAME: {\"GEMINI_API_KEY\"},\n DEEPSEEK_MODEL_FULLNAME : {\"GROQ_DEEPSEEK_API_KEY\"},\n LLAMA3_MODEL_FULLNAME : {\"GROQ_LLAMA_API_KEY\"},\n MISTRAL_MODEL_FULLNAME : {\"MISTRAL_API_KEY\"},\n}\n\n\n\n# Timeout settings for web scraping\nTIMEOUT_SETTINGS = {\n \"page_load\": 30,\n \"script\": 10\n}\n\nNUMBER_SCROLL=2\n\n# RAG SYSTEM INSTRUCTIONS \n\nRAG_USER_MESSAGE = f\"Extract the following information from the provided text:\\n\\n\"\n \n\nRAG_SYSTEM_MESSAGE = (\n \"Answer the question based solely on the provided context. \"\n \"If insufficient context is provided, mention that information is missing.\"\n \"If the user is saying hello, or just talking to you without any context or question, \"\n \"you can respond with a greeting or a general response.\"\n )\n\nEMBEDDING_COST_PER_1M_TOKENS = 0 # $0.10 per 1M tokens for text-embedding-004 (google embedding model)\n\n# SCRAPER SYSTEM INSTRUCTIONS\nSYSTEM_MESSAGE = \"\"\"You are an intelligent text extraction and conversion assistant. Your task is to extract structured information \n from the given text and convert it into a pure JSON format. The JSON should contain only the structured data extracted from the text, \n with no additional commentary, explanations, or extraneous information. \n You could encounter cases where you can't find the data of the fields you have to extract or the data will be in a foreign language.\n Please process the following text and provide the output in pure JSON format with no words before or after the JSON:\"\"\"\n\nUSER_MESSAGE = f\"Extract the following information from the provided text:\\nPage content:\\n\\n\"\n \n\n\n\n\nPROMPT_PAGINATION = \"\"\"\nYou are an assistant that extracts pagination URLs from markdown content of websites. \nYour task is to identify and generate a list of pagination URLs based on a detected URL pattern where page numbers increment sequentially. Follow these instructions carefully:\n\n-Identify the Pagination Pattern:\nAnalyze the provided markdown text to detect URLs that follow a pattern where only a numeric page indicator changes.\nIf the numbers start from a low value and increment, generate the full sequence of URLs—even if not all numbers are present in the text.\n\n-Construct Complete URLs:\nIn cases where only part of a URL is provided, combine it with the given base URL (which will appear at the end of this prompt) to form complete URLs.\nEnsure that every URL you generate is clickable and leads directly to the intended page.\n\n-Incorporate User Indications:\nIf additional user instructions about the pagination mechanism are provided at the end of the prompt, use those instructions to refine your URL generation.\nOutput Format Requirements:\n\n-Strictly output only a valid JSON object with the exact structure below:\n\"\"\n{\n \"page_urls\": [\"url1\", \"url2\", \"url3\", ..., \"urlN\"]\n}\"\"\n\n\nIMPORTANT:\n\nOutput only a single valid JSON object with no additional text, markdown formatting, or explanation.\nDo not include any extra newlines or spaces before or after the JSON.\nThe JSON object must exactly match the following schema:\n\"\"\"\n"}}}} |