{"repo": "abhij1306/feedllama", "files": {"llm_version/feed_optimizer_openrouter.py": {"88926ea20be6ae89d954185213a5af15078bef72": {"t": 1785596014.787115, "n": 22803, "h": "bea3f0d756d61801", "c": "import streamlit as st\nimport pandas as pd\nimport requests\nimport json\nimport re\nimport os\nfrom dotenv import load_dotenv\n\n# Load environment variables from .env file\nload_dotenv()\n\n# ============================================================\n# File loading helpers\n# ============================================================\n\ndef load_table(uploaded_file):\n if uploaded_file is None:\n return None\n name = uploaded_file.name.lower()\n if name.endswith(\".csv\") or name.endswith(\".txt\"):\n return pd.read_csv(uploaded_file)\n elif name.endswith(\".xls\") or name.endswith(\".xlsx\"):\n return pd.read_excel(uploaded_file)\n else:\n raise ValueError(f\"Unsupported file type: {name}\")\n\n\ndef load_taxonomy(uploaded_file):\n if uploaded_file is None:\n return []\n content = uploaded_file.read().decode(\"utf-8\")\n lines = [ln.strip() for ln in content.split(\"\\n\")]\n return [ln for ln in lines if ln]\n\n\n# ============================================================\n# Extract product types from existing taxonomy columns\n# ============================================================\n\ndef extract_product_keywords(df, title_column='title', max_keywords=500):\n \"\"\"\n Extract unique product types using pdxtstyletax and pdxtcat_testing columns.\n Preserves exact values as they appear in the data.\n \"\"\"\n \n # Check for required columns\n has_styletax = 'pdxtstyletax' in df.columns\n has_cat_testing = 'pdxtcat_testing' in df.columns\n \n if not has_styletax and not has_cat_testing:\n st.error(\"Required columns 'pdxtstyletax' or 'pdxtcat_testing' not found in data!\")\n st.info(f\"Available columns: {', '.join(df.columns.tolist())}\")\n raise ValueError(\"Missing required taxonomy columns\")\n \n st.info(f\"Using columns: pdxtstyletax={has_styletax}, pdxtcat_testing={has_cat_testing}\")\n \n # Find title column\n if title_column not in df.columns:\n possible_cols = [col for col in df.columns if 'title' in col.lower() or 'name' in col.lower() or 'product' in col.lower()]\n if possible_cols:\n title_column = possible_cols[0]\n else:\n title_column = None\n \n # Extract unique values from taxonomy columns (preserve exact casing and format)\n product_types = set()\n \n # Primary source: pdxtstyletax\n if has_styletax:\n styletax_values = df['pdxtstyletax'].dropna().astype(str).unique()\n for val in styletax_values:\n val = val.strip()\n if val and val not in ['', 'nan', 'NaN', 'none', 'None', 'null', 'NULL']:\n product_types.add(val) # Keep original casing\n st.info(f\"Found {len(styletax_values)} unique values in pdxtstyletax\")\n \n # Secondary source: pdxtcat_testing (for second tier)\n if has_cat_testing:\n cat_values = df['pdxtcat_testing'].dropna().astype(str).unique()\n for val in cat_values:\n val = val.strip()\n if val and val not in ['', 'nan', 'NaN', 'none', 'None', 'null', 'NULL']:\n product_types.add(val) # Keep original casing\n st.info(f\"Found {len(cat_values)} unique values in pdxtcat_testing\")\n \n # Convert to list and sort\n unique_keywords = sorted(list(product_types))[:max_keywords]\n \n st.success(f\"Extracted {len(unique_keywords)} unique product types from taxonomy columns\")\n \n return unique_keywords, title_column if title_column else 'title'\n\n\ndef create_sample_with_keywords(df, keywords, title_column, samples_per_keyword=2):\n \"\"\"\n Create a representative sample showing products for each keyword.\n Searches in pdxtstyletax and pdxtcat_testing columns for taxonomy keywords.\n \"\"\"\n samples = []\n \n # Check which columns are available\n has_styletax = 'pdxtstyletax' in df.columns\n has_cat_testing = 'pdxtcat_testing' in df.columns\n \n for kw in keywords[:100]: # Limit to top 100 keywords for the sample\n # Search in taxonomy columns if available\n if has_styletax:\n matching = df[df['pdxtstyletax'].str.contains(kw, case=False, na=False, regex=False)]\n elif has_cat_testing:\n matching = df[df['pdxtcat_testing'].str.contains(kw, case=False, na=False, regex=False)]\n else:\n # Fallback to title if taxonomy columns not found\n matching = df[df[title_column].str.contains(kw, case=False, na=False, regex=False)]\n \n if not matching.empty:\n samples.append(matching.head(samples_per_keyword))\n \n if samples:\n return pd.concat(samples).drop_duplicates()\n return df.head(100)\n\n\n# ============================================================\n# OpenRouter chat completion\n# ============================================================\n\ndef call_openrouter(model, system_prompt, user_prompt, api_key):\n url = \"https://openrouter.ai/api/v1/chat/completions\"\n\n headers = {\n \"Authorization\": f\"Bearer {api_key}\",\n \"Content-Type\": \"application/json\",\n \"HTTP-Referer\": \"https://localhost\",\n \"X-Title\": \"FeedOptimizer\"\n }\n\n payload = {\n \"model\": model,\n \"messages\": [\n {\"role\": \"system\", \"content\": system_prompt},\n {\"role\": \"user\", \"content\": user_prompt}\n ]\n }\n\n try:\n resp = requests.post(url, json=payload, headers=headers, timeout=600)\n resp.raise_for_status()\n return resp.json()[\"choices\"][0][\"message\"][\"content\"]\n except requests.exceptions.HTTPError as e:\n error_detail = \"\"\n try:\n error_detail = resp.json()\n except:\n error_detail = resp.text\n raise Exception(f\"OpenRouter API Error ({resp.status_code}): {error_detail}\") from e\n\n\n# ============================================================\n# JSON array extraction\n# ============================================================\n\ndef extract_json_array(text):\n first = text.find(\"[\")\n last = text.rfind(\"]\")\n if first == -1 or last == -1:\n raise ValueError(\"LLM output did not contain a JSON array.\")\n return json.loads(text[first:last+1])\n\n\n# ============================================================\n# Prompt builders\n# ============================================================\n\ndef build_pt_prompts(keywords, sample_df, instr_df, title_column):\n # Create a compact sample JSON showing both taxonomy columns and title\n sample_json = sample_df.head(100).to_json(orient=\"records\", force_ascii=False)\n instr_json = instr_df.head(20).to_json(orient=\"records\", force_ascii=False) if instr_df is not None else \"[]\"\n \n keywords_text = \"\\n\".join([f\"- {kw}\" for kw in keywords[:200]])\n\n system_prompt = \"\"\"\nYou are a product feed optimizer generating transformation rules for `product_type` (Feedonomics format).\n\nIMPORTANT CONTEXT:\n- Primary source: pdxtstyletax column (specific product styles like \"Plain Denim Jacket\")\n- Secondary source: pdxtcat_testing column (subcategories like \"Jackets & Coats\")\n- Validation: You MUST check if pdxtstyletax logically matches the product title\n- Fallback: If pdxtstyletax doesn't match title, extract keywords from title instead\n- CRITICAL: The final product type in the hierarchy MUST be PLURALIZED\n- CRITICAL: The \"then\" value MUST be wrapped in single quotes\n- CRITICAL: export_id is ALWAYS 0, enabled is ALWAYS \"1\"\n- CRITICAL: Condition must check BOTH pdxtstyletax AND pdxtcat_testing columns\n\nVALIDATION LOGIC:\n1. For each product, compare pdxtstyletax value with the title\n2. If they match logically (e.g., \"Plain Denim Jacket\" in title contains \"jacket\" or \"denim\"):\n - Use pdxtstyletax value for the condition\n - PLURALIZE it for the hierarchy (e.g., \"Plain Denim Jacket\" → \"Plain Denim Jackets\")\n - Build hierarchy: \"Category > pdxtcat_testing > Pluralized pdxtstyletax\"\n - Mark as MATCH in validation_status\n3. If they DON'T match (e.g., pdxtstyletax=\"Dress\" but title=\"Men's Cotton Shirt\"):\n - Extract the actual product type from title (e.g., \"Casual Shirt\")\n - PLURALIZE it for the hierarchy (e.g., \"Casual Shirt\" → \"Casual Shirts\")\n - Build hierarchy: \"Category > Subcategory > Pluralized Keyword\"\n - Mark as MISMATCH in validation_status\n\nRULE STRUCTURE (EXACT FORMAT):\n{\n \"field_name\": \"product_type\",\n \"export_id\": 0,\n \"if\": \"[product_type] equal '' AND [pdxtstyletax] contains '' AND [pdxtcat_testing] contains ''\",\n \"then\": \"'Category > Subcategory > '\",\n \"enabled\": \"1\",\n \"validation_status\": \"MATCH\" or \"MISMATCH\"\n}\n\nEXAMPLES:\n\nExample 1 (Match - use pdxtstyletax):\nTitle: \"Women's Plain Denim Jacket Blue\"\npdxtstyletax: \"Plain Denim Jacket\"\npdxtcat_testing: \"Jackets & Coats\"\n✓ Match! Output:\n{\n \"field_name\": \"product_type\",\n \"export_id\": 0,\n \"if\": \"[product_type] equal '' AND [pdxtstyletax] contains 'Plain Denim Jacket' AND [pdxtcat_testing] contains 'Plain Denim Jacket'\",\n \"then\": \"'Clothing > Jackets & Coats > Plain Denim Jackets'\",\n \"enabled\": \"1\",\n \"validation_status\": \"MATCH\"\n}\n\nExample 2 (Mismatch - extract from title):\nTitle: \"Men's Casual Cotton Shirt White\"\npdxtstyletax: \"Dress\"\npdxtcat_testing: \"Shirts & Tops\"\n✗ Mismatch! Extract \"Casual Shirt\" from title. Output:\n{\n \"field_name\": \"product_type\",\n \"export_id\": 0,\n \"if\": \"[product_type] equal '' AND [title] contains 'Casual Shirt' AND [pdxtcat_testing] contains 'Casual Shirt'\",\n \"then\": \"'Clothing > Shirts & Tops > Casual Shirts'\",\n \"enabled\": \"1\",\n \"validation_status\": \"MISMATCH\"\n}\n\nPLURALIZATION RULES:\n- Add 's' to most words: \"Jacket\" → \"Jackets\", \"Shirt\" → \"Shirts\"\n- Words ending in 's', 'x', 'z', 'ch', 'sh': add 'es': \"Dress\" → \"Dresses\"\n- Words ending in consonant + 'y': change 'y' to 'ies': \"Accessory\" → \"Accessories\"\n- Irregular plurals: \"Scarf\" → \"Scarves\", \"Leaf\" → \"Leaves\"\n- Keep compound words intact: \"Plain Denim Jacket\" → \"Plain Denim Jackets\"\n\nCRITICAL REQUIREMENTS:\n- Validate EVERY pdxtstyletax value against its title\n- Use pdxtstyletax when it matches, extract from title when it doesn't\n- For mismatches, use [title] in the condition instead of [pdxtstyletax]\n- ALWAYS include BOTH [pdxtstyletax]/[title] AND [pdxtcat_testing] in the condition\n- ALWAYS pluralize the final product type in the hierarchy\n- Keep singular form in the condition, plural in the output\n- ALWAYS wrap the \"then\" value in single quotes: 'Category > Subcategory > Product'\n- ALWAYS use export_id: 0 and enabled: \"1\" for every rule\n- ALWAYS include validation_status field: \"MATCH\" or \"MISMATCH\"\n- Always preserve exact casing for keywords\n- Output ONLY a JSON array\n- No markdown, no explanation\n\"\"\"\n\n user_prompt = f\"\"\"\nPRODUCT STYLES FROM pdxtstyletax (total {len(keywords)} styles):\n{keywords_text}\n\nSAMPLE DATA (showing pdxtstyletax, pdxtcat_testing, and title):\n{sample_json[:5000]}\n\nTASK:\n1. For EACH unique pdxtstyletax value, find sample products with that value\n2. Check if pdxtstyletax logically matches the product titles\n3. If MATCH: Create rule using pdxtstyletax (singular in condition, PLURAL in hierarchy)\n4. If MISMATCH: Extract correct keyword from title (singular in condition, PLURAL in hierarchy)\n\nREMEMBER: \n- Wrap the \"then\" value in single quotes like: 'Clothing > Category > Product'\n- Use export_id: 0 and enabled: \"1\" for ALL rules\n- Include BOTH columns in condition: [pdxtstyletax] AND [pdxtcat_testing] (or [title] AND [pdxtcat_testing] for mismatches)\n- Add validation_status field to track matches vs mismatches\n\nGenerate product_type rules for ALL {len(keywords[:200])} styles above.\nReturn ONLY a pure JSON array.\n\"\"\"\n return system_prompt, user_prompt\n\n\ndef build_gpc_prompts(pt_rules, taxonomy):\n pt_json = json.dumps(pt_rules, ensure_ascii=False)\n taxonomy_text = \"\\n\".join(taxonomy[:300])\n\n system_prompt = \"\"\"\nYou generate google_product_category rules.\n\nSteps:\n1. For EVERY product_type rule provided, create a corresponding GPC rule\n2. Extract the main product keyword from each product_type\n3. Match to the closest valid Google Product Taxonomy category\n4. Output Feedonomics-compatible rules:\n\n- field_name: \"google_product_category\"\n- export_id: 0 (always 0)\n- if: \"[google_product_category] equal '' AND [product_type] contains 'keyword'\"\n- then: 'EXACT taxonomy category' (wrapped in single quotes)\n- enabled: \"1\"\n\nCRITICAL REQUIREMENTS:\n- Generate a GPC rule for EVERY product_type rule provided\n- Match to valid taxonomy categories only\n- ALWAYS wrap the \"then\" value in single quotes: 'Category > Subcategory'\n- ALWAYS use export_id: 0 and enabled: \"1\"\n- Output MUST be a JSON array only\n- No markdown, no explanation\n\"\"\"\n\n user_prompt = f\"\"\"\nPRODUCT TYPE RULES (total {len(pt_rules)} rules):\n{pt_json[:20000]}\n\nGOOGLE PRODUCT TAXONOMY:\n{taxonomy_text}\n\nIMPORTANT: Generate google_product_category rules for ALL {len(pt_rules)} product_type rules.\nMatch each product type to the most appropriate Google taxonomy category.\nREMEMBER: Wrap the \"then\" value in single quotes like: 'Apparel & Accessories > Clothing > Shirts'\n\nReturn ONLY a pure JSON array.\n\"\"\"\n\n return system_prompt, user_prompt\n\n\n# ============================================================\n# Rule generators\n# ============================================================\n\ndef generate_pt_rules(raw_df, instr_df, model, api_key):\n # Extract unique keywords from all rows\n with st.spinner(\"Analyzing all 85,000+ products to extract unique keywords...\"):\n keywords, title_column = extract_product_keywords(raw_df, max_keywords=500)\n st.success(f\"Found {len(keywords)} unique product keywords across all rows\")\n \n # Create representative sample\n sample_df = create_sample_with_keywords(raw_df, keywords, title_column)\n \n # Build prompts with keywords\n sys_prompt, usr_prompt = build_pt_prompts(keywords, sample_df, instr_df, title_column)\n \n # Call LLM\n with st.spinner(f\"Generating rules for {len(keywords)} product types...\"):\n text = call_openrouter(model, sys_prompt, usr_prompt, api_key)\n \n rules = extract_json_array(text)\n\n cleaned = []\n match_count = 0\n mismatch_count = 0\n \n for i, r in enumerate(rules):\n cond = str(r.get(\"if\", \"\")).strip()\n then_val = str(r.get(\"then\", \"\")).strip()\n validation_status = str(r.get(\"validation_status\", \"UNKNOWN\")).strip()\n \n if cond and then_val:\n cleaned.append({\n \"field_name\": \"product_type\",\n \"export_id\": i,\n \"if\": cond,\n \"then\": then_val,\n \"enabled\": \"1\",\n })\n \n # Track validation status\n if validation_status == \"MATCH\":\n match_count += 1\n elif validation_status == \"MISMATCH\":\n mismatch_count += 1\n \n # Display discrepancy statistics\n if match_count > 0 or mismatch_count > 0:\n st.info(f\"📊 Validation Results: {match_count} matches, {mismatch_count} discrepancies found\")\n \n return cleaned\n\n\ndef generate_gpc_rules(pt_rules, taxonomy, model, api_key):\n \"\"\"\n Generate GPC rules by extracting base nouns from product_type rules.\n Groups multiple nouns that map to the same category using contains_any.\n \"\"\"\n \n # Extract unique base nouns from product_type rules\n base_nouns = set()\n for rule in pt_rules:\n if_condition = rule.get('if', '')\n # Extract the keyword from the condition\n # Format: \"[product_type] equal '' AND [title] contains 'keyword'\"\n import re\n match = re.search(r\"contains ['\\\"]([^'\\\"]+)['\\\"]\", if_condition)\n if match:\n keyword = match.group(1).strip()\n # Remove adjectives - take the last word (the noun)\n words = keyword.split()\n base_noun = words[-1] if words else keyword\n base_nouns.add(base_noun)\n \n base_nouns_list = sorted(list(base_nouns))\n \n st.info(f\"Extracted {len(base_nouns_list)} unique base nouns for GPC mapping\")\n \n # Now ask LLM to map these base nouns to Google taxonomy\n taxonomy_text = \"\\n\".join(taxonomy[:500])\n nouns_text = \"\\n\".join([f\"- {noun}\" for noun in base_nouns_list])\n \n system_prompt = \"\"\"\nYou are mapping apparel product nouns to Google Product Taxonomy categories.\n\nFor EACH noun provided, find the MOST SPECIFIC matching category from the Google Product Taxonomy.\n\nOutput format: JSON array with objects containing:\n- noun: the product noun\n- category: the EXACT Google Product Taxonomy category\n\nCRITICAL:\n- Map EVERY noun provided\n- Use EXACT taxonomy categories only\n- Output ONLY a JSON array\n- No markdown, no explanation\n\"\"\"\n\n user_prompt = f\"\"\"\nPRODUCT NOUNS TO MAP (total {len(base_nouns_list)}):\n{nouns_text}\n\nGOOGLE PRODUCT TAXONOMY:\n{taxonomy_text}\n\nMap EVERY noun above to its most appropriate Google Product Taxonomy category.\nReturn ONLY a JSON array: [{{\"noun\": \"shirt\", \"category\": \"Apparel & Accessories > Clothing > Shirts & Tops\"}}, ...]\n\"\"\"\n\n # Call LLM for mapping\n with st.spinner(f\"Mapping {len(base_nouns_list)} product nouns to Google taxonomy...\"):\n text = call_openrouter(model, system_prompt, user_prompt, api_key)\n \n mappings = extract_json_array(text)\n \n # Validate and group nouns by category\n taxonomy_set = set(taxonomy)\n category_to_nouns = {} # category -> list of nouns\n \n for mapping in mappings:\n noun = str(mapping.get(\"noun\", \"\")).strip()\n category = str(mapping.get(\"category\", \"\")).strip()\n \n if not noun or not category:\n continue\n \n # Validate category exists in taxonomy\n if category not in taxonomy_set:\n # Fuzzy match fallback\n matches = [t for t in taxonomy if category.lower() in t.lower() or noun.lower() in t.lower()]\n if matches:\n category = matches[0]\n else:\n # Skip if no match found\n continue\n \n # Group nouns by category\n if category not in category_to_nouns:\n category_to_nouns[category] = []\n category_to_nouns[category].append(noun)\n \n # Create GPC rules with contains_any for multiple nouns\n cleaned = []\n export_id = 0\n \n for category, nouns in sorted(category_to_nouns.items()):\n if len(nouns) == 1:\n # Single noun - use simple contains\n cleaned.append({\n \"field_name\": \"google_product_category\",\n \"export_id\": export_id,\n \"if\": f\"[google_product_category] equal '' AND [product_type] contains '{nouns[0]}'\",\n \"then\": f\"'{category}'\",\n \"enabled\": \"1\",\n })\n else:\n # Multiple nouns - use contains_any\n nouns_str = \", \".join([f\"'{noun}'\" for noun in sorted(nouns)])\n cleaned.append({\n \"field_name\": \"google_product_category\",\n \"export_id\": export_id,\n \"if\": f\"[google_product_category] equal '' AND [product_type] contains_any ({nouns_str})\",\n \"then\": f\"'{category}'\",\n \"enabled\": \"1\",\n })\n export_id += 1\n \n st.success(f\"Grouped {len(base_nouns_list)} nouns into {len(cleaned)} GPC rules\")\n \n return cleaned\n\n\n# ============================================================\n# STREAMLIT UI\n# ============================================================\n\nst.set_page_config(page_title=\"Feed Optimizer (OpenRouter)\", layout=\"wide\")\n\nst.title(\"🧠 Feed Optimizer – Rule Generator (OpenRouter Version)\")\nst.write(\"Generate Product Type & GPC Rules using OpenRouter models.\")\nst.info(\"✨ This version analyzes ALL rows in your dataset to extract unique product types, then generates comprehensive rules.\")\n\nwith st.sidebar:\n st.header(\"🔑 API Settings\")\n \n # Try to load API key from environment variable\n env_api_key = os.getenv(\"OPENROUTER_API_KEY\", \"\")\n \n if env_api_key and env_api_key != \"your_api_key_here\":\n api_key = env_api_key\n st.success(\"✅ API key loaded from .env file\")\n else:\n api_key = st.text_input(\"OpenRouter API Key\", type=\"password\", help=\"Or add OPENROUTER_API_KEY to .env file\")\n \n model = \"kwaipilot/kat-coder-pro-v1:free\" # Using Kwaipilot free model\n st.info(f\"Using model: {model}\")\n\nst.subheader(\"1. Upload Files\")\n\nraw_file = st.file_uploader(\"Raw Data (CSV/Excel)\", type=[\"csv\", \"xls\", \"xlsx\", \"txt\"])\ninstr_file = st.file_uploader(\"Client Instructions (CSV/Excel)\", type=[\"csv\", \"xls\", \"xlsx\"])\n\n# Load GPC taxonomy from local file\nTAXONOMY_FILE = \"google_taxonomy.txt\"\nif not os.path.exists(TAXONOMY_FILE):\n st.error(f\"❌ Google taxonomy file not found: {TAXONOMY_FILE}\")\n st.info(\"Please ensure google_taxonomy.txt is in the same directory as this script.\")\n st.stop()\n\nst.success(f\"✅ Using Google taxonomy file: {TAXONOMY_FILE}\")\n\nstart = st.button(\"🚀 Generate Rules\")\n\n\nif start:\n if not api_key:\n st.error(\"Please enter your OpenRouter API key.\")\n st.stop()\n\n if raw_file is None:\n st.error(\"Please upload Raw Data file.\")\n st.stop()\n\n with st.spinner(\"Loading files…\"):\n raw_df = load_table(raw_file)\n instr_df = load_table(instr_file) if instr_file else None\n \n # Load taxonomy from local file\n with open(TAXONOMY_FILE, 'r', encoding='utf-8') as f:\n taxonomy = [line.strip() for line in f if line.strip()]\n\n st.success(f\"Files loaded. Raw data contains {len(raw_df):,} rows.\")\n\n # PRODUCT TYPE RULES\n st.subheader(\"2. Generating Product Type Rules…\")\n try:\n pt_rules = generate_pt_rules(raw_df, instr_df, model, api_key)\n except Exception as e:\n st.error(\"Error generating product type rules.\")\n st.exception(e)\n st.stop()\n\n df_pt = pd.DataFrame(pt_rules)\n st.success(f\"Generated {len(pt_rules)} product type rules!\")\n st.write(df_pt)\n\n pt_csv = df_pt.to_csv(index=False).encode(\"utf-8\")\n st.download_button(\"Download Product Type Rules CSV\", pt_csv, \"product_type_rules.csv\", \"text/csv\")\n\n # GPC RULES\n st.subheader(\"3. Generating Google Product Category Rules…\")\n with st.spinner(\"OpenRouter model generating GPC rules…\"):\n try:\n gpc_rules = generate_gpc_rules(pt_rules, taxonomy, model, api_key)\n except Exception as e:\n st.error(\"Error generating GPC rules.\")\n st.exception(e)\n st.stop()\n\n df_gpc = pd.DataFrame(gpc_rules)\n st.success(f\"Generated {len(gpc_rules)} GPC rules!\")\n st.write(df_gpc)\n\n gpc_csv = df_gpc.to_csv(index=False).encode(\"utf-8\")\n st.download_button(\"Download GPC Rules CSV\", gpc_csv, \"gpc_rules.csv\", \"text/csv\")\n\n st.success(\"All rules generated successfully!\")\n"}}}}