1 line
68 KiB
JSON
1 line
68 KiB
JSON
{"repo": "automatesolutions/bayanwin", "files": {"backend/app.py": {"7c58ab0098df25c78fd32c456cedf25590eab7b0": {"t": 1785596424.0449948, "n": 66957, "h": "482ffbe25838ba09", "c": "\"\"\"FastAPI application with API endpoints - Using InstantDB only.\"\"\"\nfrom fastapi import FastAPI, HTTPException, Query, Path, Body, Request, Header, BackgroundTasks\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom fastapi.responses import JSONResponse, StreamingResponse\nfrom datetime import datetime, date, timedelta\nfrom services.instantdb_client import instantdb\nfrom scrapers.google_sheets_scraper import GoogleSheetsScraper\nfrom ml_models.xgboost_model import XGBoostModel\nfrom ml_models.decision_tree import DecisionTreeModel\nfrom ml_models.markov_chain import MarkovChainModel\nfrom ml_models.anomaly_detection import AnomalyDetectionModel\nfrom ml_models.drl_agent import DRLAgent\nfrom ml_models.nash_hot_filter import NashHotFilterModel\nfrom utils.frequency_analysis import get_hot_numbers, get_cold_numbers, get_overdue_numbers, calculate_frequency\nfrom utils.error_distance_calculator import calculate_all_metrics\nfrom utils.graph_aggregates import cooccurrence_edges, markov_ball_transitions, sankey_hot_model_votes\nfrom utils.ttl_cache import get_or_set\nfrom services.apify_ingest import ingest_apify_run, auto_ingest_from_apify_actor\nfrom services.prediction_council import run_council_report, load_latest_predictions_for_council\nfrom services.memory_service import (\n build_memory_context_for_council,\n get_user_memory_record,\n upsert_user_memory,\n)\nfrom config import Config\nfrom typing import Optional, Any, Dict, List, Tuple\nfrom bisect import bisect_left\nfrom pydantic import BaseModel\nfrom openai import APIConnectionError, AuthenticationError, RateLimitError\nimport concurrent.futures\nimport asyncio\nimport json\nimport logging\nimport numpy as np\nimport queue\nimport threading\n\nfrom services.miro_strategy import run_miro_strategy_predict\n\n# Configure logging to show INFO level messages\nlogging.basicConfig(\n level=logging.INFO,\n format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'\n)\nlogger = logging.getLogger(__name__)\n\n# Short TTL JSON caches for heavy dashboard endpoints (per process; cleared on reload)\n_stats_api_cache: Dict[Any, Any] = {}\n_gaussian_api_cache: Dict[Any, Any] = {}\n\n\ndef _prev_preds_lists_from_rows(all_rows: List[Dict[str, Any]], model_name: str) -> List[Any]:\n \"\"\"Up to 5 prior numeric rows for this model_type from a shared get_predictions page.\"\"\"\n prev_preds: List[Any] = []\n for prev_pred in all_rows:\n if prev_pred.get('model_type') != model_name:\n continue\n prev_preds.append(\n [\n prev_pred.get('predicted_number_1'),\n prev_pred.get('predicted_number_2'),\n prev_pred.get('predicted_number_3'),\n prev_pred.get('predicted_number_4'),\n prev_pred.get('predicted_number_5'),\n prev_pred.get('predicted_number_6'),\n ]\n )\n if len(prev_preds) >= 5:\n break\n while len(prev_preds) < 5:\n prev_preds.append(None)\n return prev_preds[:5]\n\n\ndef _http_exception_from_llm_error(exc: Exception) -> HTTPException:\n \"\"\"Map OpenAI client errors to HTTP responses with actionable messages.\"\"\"\n if isinstance(exc, AuthenticationError):\n return HTTPException(\n status_code=401,\n detail=(\n \"LLM API key was rejected. For OpenAI, keys normally start with sk- or sk-proj-. \"\n \"If yours looks like k-proj-..., add the missing leading s. \"\n \"Check LLM_API_KEY and LLM_BASE_URL in .env, then restart the server.\"\n ),\n )\n if isinstance(exc, RateLimitError):\n return HTTPException(\n status_code=429,\n detail=\"LLM rate limit exceeded. Try again in a few minutes.\",\n )\n if isinstance(exc, APIConnectionError):\n return HTTPException(\n status_code=502,\n detail=f\"Could not reach LLM API: {exc!s}\",\n )\n return HTTPException(status_code=500, detail=str(exc))\n\n\ndef convert_to_python_types(obj):\n \"\"\"Convert numpy types to native Python types for JSON serialization.\"\"\"\n if isinstance(obj, dict):\n return {k: convert_to_python_types(v) for k, v in obj.items()}\n elif isinstance(obj, list):\n return [convert_to_python_types(item) for item in obj]\n elif isinstance(obj, (np.integer, np.int64, np.int32)):\n return int(obj)\n elif isinstance(obj, (np.floating, np.float64, np.float32)):\n return float(obj)\n elif isinstance(obj, np.ndarray):\n return obj.tolist()\n else:\n return obj\n\n# Initialize FastAPI app\napp = FastAPI(\n title=\"Lotto Prediction API\",\n description=\"API for lottery predictions using ML models - InstantDB only\",\n version=\"1.0.0\"\n)\n\n# Configure CORS\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"], # In production, specify actual origins\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n\n# Initialize ML models\nxgboost_model = XGBoostModel()\ndecision_tree_model = DecisionTreeModel()\nmarkov_chain_model = MarkovChainModel()\nanomaly_detection_model = AnomalyDetectionModel()\ndrl_agent = DRLAgent()\nnash_hot_filter_model = NashHotFilterModel(mode='hybrid')\n\n# Pydantic models for request/response validation\nclass ScrapeRequest(BaseModel):\n # Date parameters not needed for Google Sheets (reads all data)\n # Kept for API compatibility\n start_date: Optional[str] = None\n end_date: Optional[str] = None\n game_type: Optional[str] = None # If specified, only scrape this game\n full_sync: Optional[bool] = False # Full CSV + reset ingest cursor (reconciliation)\n\nclass CalculateAccuracyRequest(BaseModel):\n result_id: str # Changed to string for InstantDB IDs\n game_type: str\n\nclass AutoCalculateAccuracyRequest(BaseModel):\n game_type: Optional[str] = None\n\n\nclass ApifyIngestRequest(BaseModel):\n run_id: str\n game_type: Optional[str] = None\n\n\nclass CouncilReportRequest(BaseModel):\n predictions: Optional[Dict[str, Any]] = None\n use_latest: bool = False\n user_key: Optional[str] = None\n\n\nclass MemoryUpsertRequest(BaseModel):\n user_key: str\n pinned_games: Optional[str] = None\n preferences: Optional[str] = None\n last_summary: Optional[str] = None\n\n\nclass SankeyRequest(BaseModel):\n predictions: Dict[str, Any]\n\n\n@app.get(\"/api/games\")\nasync def get_games():\n \"\"\"List available games.\"\"\"\n games = []\n for game_id, game_info in Config.GAMES.items():\n games.append({\n 'id': game_id,\n 'name': game_info['name'],\n 'min_number': game_info['min_number'],\n 'max_number': game_info['max_number'],\n 'numbers_count': game_info['numbers_count']\n })\n return games\n\n@app.get(\"/api/results/{game_type}\")\nasync def get_results(\n game_type: str = Path(..., description=\"Game type identifier\"),\n page: int = Query(1, ge=1, description=\"Page number\"),\n limit: int = Query(50, ge=1, le=100, description=\"Items per page\")\n):\n \"\"\"Get historical results for a game.\"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n \n offset = (page - 1) * limit\n \n results, total, has_more = instantdb.get_results_with_total(\n game_type, limit=limit, offset=offset, order_by='draw_date.desc'\n )\n \n results_data = []\n for result in results:\n results_data.append({\n 'id': result.get('id'),\n 'draw_date': result.get('draw_date'),\n 'draw_number': result.get('draw_number'),\n 'numbers': [\n result.get('number_1'), result.get('number_2'), result.get('number_3'),\n result.get('number_4'), result.get('number_5'), result.get('number_6')\n ],\n 'jackpot': result.get('jackpot'),\n 'winners': result.get('winners'),\n 'created_at': result.get('created_at')\n })\n \n out: Dict[str, Any] = {\n 'results': results_data,\n 'page': page,\n 'limit': limit,\n 'has_more': has_more,\n }\n if total is not None:\n out['total'] = total\n return out\n\nasync def auto_calculate_accuracy_for_new_results(\n game_type: str = None,\n *,\n results_limit: Optional[int] = None,\n predictions_limit: Optional[int] = None,\n):\n \"\"\"\n Automatically match predictions to results and calculate accuracy.\n This runs after scraping to populate the Error Distance Analysis dashboard.\n Use results_limit / predictions_limit to cap work (e.g. post-scrape: recent rows only).\n \"\"\"\n games_to_process = [game_type] if game_type else Config.GAMES.keys()\n rl = results_limit if results_limit is not None else Config.ACCURACY_AUTO_RESULTS_LIMIT\n pl = predictions_limit if predictions_limit is not None else Config.ACCURACY_AUTO_PREDICTIONS_LIMIT\n\n total_calculated = 0\n\n for game in games_to_process:\n try:\n # Newest-first; limit avoids scanning the full history after each small sheet update\n all_results = instantdb.get_results(game, limit=rl, offset=0, order_by='draw_date.desc')\n valid_results = []\n\n for r in all_results:\n draw_date = r.get('draw_date')\n if not draw_date:\n continue\n\n # Validate that result has all required numbers\n if not all(r.get(f'number_{i}') for i in range(1, 7)):\n continue\n\n valid_results.append(r)\n\n if not valid_results:\n logger.info(f\"No valid results found for {game}\")\n continue\n\n logger.info(f\"Found {len(valid_results)} valid results for {game} (limit={rl})\")\n\n predictions = instantdb.get_predictions(game, limit=pl)\n logger.info(f\"Found {len(predictions)} predictions for {game} (limit={pl})\")\n\n candidate_result_ids = {str(r.get('id')) for r in valid_results if r.get('id')}\n existing_accuracy = instantdb.get_prediction_accuracy(game)\n existing_pairs = set()\n for acc in existing_accuracy:\n pred_id = acc.get('prediction_id')\n res_id = acc.get('result_id')\n if pred_id and res_id:\n sid = str(res_id)\n if sid in candidate_result_ids:\n existing_pairs.add((str(pred_id), sid))\n\n logger.info(\n \"Considering %s existing accuracy pair(s) for this result window (of %s loaded)\",\n len(existing_pairs),\n len(existing_accuracy),\n )\n calculated_count = 0\n \n # Helper function to parse date strings\n def parse_date_str(s):\n if not s:\n return None\n d = s.split('T')[0] if 'T' in s else s[:10]\n try:\n return datetime.strptime(d, '%Y-%m-%d').date()\n except:\n return None\n \n # Pre-parse result dates\n results_with_date = [\n (r, parse_date_str(r.get('draw_date')))\n for r in valid_results\n ]\n results_with_date = [(r, d) for (r, d) in results_with_date if d is not None]\n\n # Predictions sorted by prediction date — use bisect instead of scanning all preds per result\n pred_dated: List[Tuple[date, Any]] = []\n for p in predictions:\n td = p.get('target_draw_date')\n if not td:\n ca = p.get('created_at')\n if ca:\n td = ca.split('T')[0] if 'T' in ca else ca[:10]\n logger.debug(\n \"Using created_at as fallback for prediction %s: %s\",\n p.get('id'),\n td,\n )\n else:\n continue\n pred_date_obj = parse_date_str(td)\n if pred_date_obj:\n pred_dated.append((pred_date_obj, p))\n pred_dated.sort(key=lambda x: x[0])\n pred_dates_sorted = [x[0] for x in pred_dated]\n \n # Find matching predictions for each result\n for idx, (r, result_date_obj) in enumerate(results_with_date):\n result_id = str(r.get('id'))\n matching_predictions = []\n \n if idx < 3:\n logger.debug(\n \"Processing result #%s: draw_date=%s, result_id=%s\",\n idx + 1,\n result_date_obj,\n result_id,\n )\n\n # pred_date < result_date and within 7 calendar days => pred_date in [result-7d, result-1d]\n window_start = result_date_obj - timedelta(days=7)\n i_lo = bisect_left(pred_dates_sorted, window_start)\n i_hi = bisect_left(pred_dates_sorted, result_date_obj)\n for j in range(i_lo, i_hi):\n pred_date_obj, p = pred_dated[j]\n matching_predictions.append(p)\n logger.debug(\n \"Match: prediction %s (%s) → result %s (%s)\",\n p.get('id'),\n pred_date_obj,\n result_id,\n result_date_obj,\n )\n \n for prediction in matching_predictions:\n pred_id = str(prediction.get('id'))\n \n # Check if this pair already has accuracy calculated\n if (pred_id, result_id) in existing_pairs:\n continue\n \n # Calculate accuracy automatically\n try:\n predicted = [\n prediction.get('predicted_number_1'),\n prediction.get('predicted_number_2'),\n prediction.get('predicted_number_3'),\n prediction.get('predicted_number_4'),\n prediction.get('predicted_number_5'),\n prediction.get('predicted_number_6')\n ]\n actual = [\n r.get('number_1'), r.get('number_2'),\n r.get('number_3'), r.get('number_4'),\n r.get('number_5'), r.get('number_6')\n ]\n \n # Validate numbers\n if not all(isinstance(n, (int, float)) and n > 0 for n in predicted + actual):\n continue\n \n metrics = calculate_all_metrics(predicted, actual)\n # Convert numpy types to native Python types for JSON serialization\n metrics_clean = convert_to_python_types(metrics)\n \n accuracy_data = {\n 'prediction_id': pred_id,\n 'result_id': result_id,\n 'error_distance': float(metrics_clean.get('error_distance')),\n 'numbers_matched': int(metrics_clean.get('numbers_matched')),\n 'distance_metrics': metrics_clean,\n 'calculated_at': datetime.now().isoformat()\n }\n \n instantdb.create_prediction_accuracy(game, accuracy_data)\n existing_pairs.add((pred_id, result_id))\n calculated_count += 1\n total_calculated += 1\n \n logger.debug(\n \"Auto-calculated accuracy for %s prediction %s vs result %s\",\n game,\n pred_id,\n result_id,\n )\n except Exception as e:\n logger.warning(f\"Failed to auto-calculate accuracy for {game} prediction {pred_id} vs result {result_id}: {e}\")\n import traceback\n logger.debug(traceback.format_exc())\n continue\n \n if calculated_count > 0:\n logger.info(f\"Auto-calculated {calculated_count} accuracy records for {game}\")\n \n except Exception as e:\n logger.error(f\"Error in auto_calculate_accuracy_for_new_results for {game}: {e}\")\n import traceback\n logger.error(traceback.format_exc())\n # Continue processing other games even if one fails\n continue\n \n logger.info(f\"Auto-calculation completed. Total calculated: {total_calculated}\")\n return total_calculated\n\n\nasync def _post_scrape_accuracy_and_drl(game_type: Optional[str]) -> None:\n \"\"\"Run after scrape response is sent so the client is not blocked on accuracy + DRL.\"\"\"\n try:\n accuracy_count = await auto_calculate_accuracy_for_new_results(\n game_type,\n results_limit=Config.ACCURACY_POST_SCRAPE_RESULTS_LIMIT,\n predictions_limit=Config.ACCURACY_POST_SCRAPE_PREDICTIONS_LIMIT,\n )\n if accuracy_count > 0:\n logger.info(\n \"Post-scrape: auto-calculated %s accuracy records\",\n accuracy_count,\n )\n if game_type:\n try:\n # learn_from_accuracy_records loads predictions/results internally; pass full accuracy list\n drl_agent.learn_from_accuracy_records(\n game_type,\n instantdb.get_prediction_accuracy(game_type),\n instantdb,\n )\n except Exception as e:\n logger.warning(\n \"Failed to update DRL after post-scrape auto-calculate (non-critical): %s\",\n e,\n )\n except Exception as e:\n logger.warning(\"Post-scrape auto-calculate accuracy failed (non-critical): %s\", e)\n\n\n@app.get(\"/api/accuracy/diagnostics/{game_type}\")\nasync def get_accuracy_diagnostics(game_type: str = Path(...)):\n \"\"\"\n Get diagnostic information about predictions, results, and accuracy records.\n Helps debug why accuracy calculation might not be working.\n \"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n \n try:\n results = instantdb.get_results(game_type, limit=1000)\n predictions = instantdb.get_predictions(game_type, limit=1000)\n accuracy_records = instantdb.get_prediction_accuracy(game_type)\n \n # Sample some dates for debugging - show first AND last dates\n sample_result_dates = [r.get('draw_date') for r in results[:10] if r.get('draw_date')]\n sample_prediction_dates = [p.get('target_draw_date') or p.get('created_at') for p in predictions[:10] if p.get('target_draw_date') or p.get('created_at')]\n \n # Get date ranges\n result_dates_all = [r.get('draw_date') for r in results if r.get('draw_date')]\n prediction_dates_all = [p.get('target_draw_date') or p.get('created_at') for p in predictions if p.get('target_draw_date') or p.get('created_at')]\n \n result_date_range = None\n if result_dates_all:\n result_date_range = {\n 'earliest': min(result_dates_all),\n 'latest': max(result_dates_all)\n }\n \n prediction_date_range = None\n if prediction_dates_all:\n prediction_date_range = {\n 'earliest': min(prediction_dates_all),\n 'latest': max(prediction_dates_all)\n }\n \n # Check for overlapping dates\n overlapping_dates = []\n if result_dates_all and prediction_dates_all:\n result_dates_set = set([d.split('T')[0] for d in result_dates_all])\n prediction_dates_set = set([d.split('T')[0] for d in prediction_dates_all])\n overlapping_dates = list(result_dates_set.intersection(prediction_dates_set))[:10]\n \n return {\n 'game_type': game_type,\n 'total_results': len(results),\n 'total_predictions': len(predictions),\n 'total_accuracy_records': len(accuracy_records),\n 'sample_result_dates': sample_result_dates,\n 'sample_prediction_dates': sample_prediction_dates,\n 'result_date_range': result_date_range,\n 'prediction_date_range': prediction_date_range,\n 'overlapping_dates': overlapping_dates,\n 'dates_match': len(overlapping_dates) > 0,\n 'has_valid_results': any(all(r.get(f'number_{i}') for i in range(1, 7)) for r in results),\n 'has_valid_predictions': any(all(p.get(f'predicted_number_{i}') for i in range(1, 7)) for p in predictions),\n 'diagnosis': _diagnose_matching_issue(len(predictions), len(results), len(overlapping_dates), prediction_date_range, result_date_range)\n }\n except Exception as e:\n logger.error(f\"Error in diagnostics: {e}\")\n raise HTTPException(status_code=500, detail=str(e))\n\ndef _diagnose_matching_issue(pred_count, result_count, overlap_count, pred_range, result_range):\n \"\"\"Diagnose why predictions and results aren't matching.\"\"\"\n if pred_count == 0:\n return \"No predictions found. Generate predictions first.\"\n if result_count == 0:\n return \"No results found. Scrape lottery results first.\"\n if overlap_count > 0:\n return f\"Found {overlap_count} matching dates! Predictions should match to results.\"\n if pred_range and result_range:\n return f\"Date mismatch: Predictions are from {pred_range['earliest']} to {pred_range['latest']}, but results are from {result_range['earliest']} to {result_range['latest']}. You need results from the same time period as your predictions.\"\n return \"Unable to determine issue. Check date formats.\"\n\n@app.post(\"/api/accuracy/auto-calculate\")\nasync def trigger_auto_calculate_accuracy(\n request: AutoCalculateAccuracyRequest = Body(...)\n):\n \"\"\"\n Manually trigger auto-calculation of accuracy for matching predictions and results.\n This can be called to populate the Error Distance Analysis dashboard.\n \"\"\"\n try:\n game_type = request.game_type\n logger.info(f\"Manual trigger: Auto-calculating accuracy for {game_type or 'all games'}\")\n \n # First, get diagnostics\n if game_type:\n results = instantdb.get_results(game_type, limit=1000)\n predictions = instantdb.get_predictions(game_type, limit=1000)\n \n if len(results) == 0:\n return {\n 'success': False,\n 'total_calculated': 0,\n 'message': f'No results found for {game_type}. Please scrape results first.',\n 'diagnostics': {\n 'results_count': 0,\n 'predictions_count': len(predictions)\n }\n }\n \n if len(predictions) == 0:\n return {\n 'success': False,\n 'total_calculated': 0,\n 'message': f'No predictions found for {game_type}. Please generate predictions by clicking \"⚡ Generate Predictions\" button first. If you already generated predictions, they might be in a different game type or not saved to the database.',\n 'diagnostics': {\n 'results_count': len(results),\n 'predictions_count': 0\n }\n }\n \n total_calculated = await auto_calculate_accuracy_for_new_results(game_type)\n \n # Get additional diagnostics if no records calculated\n if total_calculated == 0 and game_type:\n results = instantdb.get_results(game_type, limit=1000)\n predictions = instantdb.get_predictions(game_type, limit=1000)\n accuracy_records = instantdb.get_prediction_accuracy(game_type)\n \n message = f'No new accuracy records calculated. '\n if len(predictions) == 0:\n message += 'No predictions found. Please generate predictions first.'\n elif len(results) == 0:\n message += 'No results found. Please scrape results first.'\n elif len(accuracy_records) > 0:\n message += f'All {len(predictions)} predictions are already matched to results.'\n else:\n message += 'Predictions and results dates do not match. The system looks for exact matches or predictions made up to 7 days before a result.'\n else:\n message = f'Successfully calculated {total_calculated} accuracy records' if total_calculated > 0 else 'No new accuracy records calculated.'\n \n return {\n 'success': True,\n 'total_calculated': total_calculated,\n 'message': message\n }\n except Exception as e:\n logger.error(f\"Error in manual auto-calculate: {e}\")\n import traceback\n error_trace = traceback.format_exc()\n logger.error(error_trace)\n # Return more detailed error message\n raise HTTPException(\n status_code=500, \n detail=f\"Failed to calculate accuracy: {str(e)}. Check server logs for details.\"\n )\n\n\nasync def _execute_sheets_scrape(\n request: ScrapeRequest,\n background_tasks: BackgroundTasks,\n full_sync: Optional[bool],\n) -> Dict[str, Any]:\n \"\"\"Shared implementation for /api/scrape and /api/cron/ingest-sheets.\"\"\"\n logger.info(\"Starting scrape operation...\")\n scraper = GoogleSheetsScraper()\n use_full_sync = bool(request.full_sync) or (full_sync is True)\n\n try:\n if request.game_type:\n if request.game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=f\"Invalid game type: {request.game_type}\")\n\n logger.info(\"Scraping single game: %s (full_sync=%s)\", request.game_type, use_full_sync)\n game_stats = await scraper.scrape_game(request.game_type, full_sync=use_full_sync)\n\n _ex = game_stats.get('existing_in_db')\n stats = {\n 'total_games': 1,\n 'games': {request.game_type: game_stats},\n 'summary': {\n 'total_results_in_sheets': game_stats.get('total_in_sheet', 0),\n 'total_existing_in_db': _ex if isinstance(_ex, int) else None,\n 'total_new_results': game_stats.get('new_results', 0),\n 'total_added': game_stats.get('added', 0)\n }\n }\n else:\n logger.info(\"Scraping all games (full_sync=%s)\", use_full_sync)\n stats = await scraper.scrape_all_games(full_sync=use_full_sync)\n\n logger.info(f\"Scrape completed. Stats: {stats}\")\n except Exception as scrape_error:\n logger.error(f\"Scrape operation failed: {scrape_error}\")\n import traceback\n logger.error(traceback.format_exc())\n raise HTTPException(\n status_code=500,\n detail=f\"Scraping failed: {str(scrape_error)}\"\n )\n\n total_added = stats.get('summary', {}).get('total_added', 0)\n has_errors = any(\n game_stats.get('error') for game_stats in stats.get('games', {}).values()\n )\n\n logger.info(f\"Scraping summary: {total_added} results added, errors: {has_errors}\")\n logger.info(f\"Game stats: {stats.get('games', {})}\")\n\n if has_errors and total_added == 0:\n error_details = {\n game: game_stats.get('error', 'Unknown error')\n for game, game_stats in stats.get('games', {}).items()\n if game_stats.get('error')\n }\n logger.error(f\"All scraping failed: {error_details}\")\n raise HTTPException(\n status_code=500,\n detail=f\"Failed to scrape data: {error_details}\"\n )\n\n games_payload = stats.get('games', {}) or {}\n sync_modes = {\n gt: (gs or {}).get('sync_mode')\n for gt, gs in games_payload.items()\n if isinstance(gs, dict) and gs.get('sync_mode')\n }\n _tie = stats.get('summary', {}).get('total_existing_in_db')\n frontend_stats = {\n 'total_new': total_added,\n 'total_duplicates': _tie if isinstance(_tie, int) else 0,\n 'games_updated': [\n game_type for game_type, game_stats in games_payload.items()\n if game_stats.get('added', 0) > 0\n ],\n 'summary': stats.get('summary', {}),\n 'games': games_payload,\n 'sync_modes': sync_modes,\n }\n\n apify_result = None\n apify_added = 0\n if getattr(Config, \"APIFY_AUTO_INGEST\", True) and Config.APIFY_API_TOKEN and getattr(Config, \"APIFY_ACTOR_ID\", None):\n try:\n apify_result = auto_ingest_from_apify_actor(request.game_type)\n if not apify_result.get(\"skipped\"):\n apify_added = int(apify_result.get(\"total_added\") or 0)\n logger.info(\"Apify auto-ingest: added %s rows (run_id=%s)\", apify_added, apify_result.get(\"run_id\"))\n except Exception as apify_err:\n logger.warning(\"Apify auto-ingest failed (non-critical): %s\", apify_err)\n apify_result = {\"skipped\": False, \"error\": str(apify_err), \"total_added\": 0}\n\n combined_added = total_added + apify_added\n\n msg_parts = []\n if total_added > 0:\n msg_parts.append(f\"Sheets: {total_added} new\")\n if apify_added > 0:\n msg_parts.append(f\"Apify: {apify_added} new\")\n if not msg_parts:\n msg_core = \"No new data to add (may already exist)\"\n else:\n msg_core = \" / \".join(msg_parts)\n\n response: Dict[str, Any] = {\n 'success': True,\n 'stats': frontend_stats,\n 'full_sync': use_full_sync,\n 'timestamp': datetime.now().isoformat(),\n 'message': msg_core,\n }\n if apify_result is not None:\n response[\"apify_ingest\"] = apify_result\n\n if combined_added == 0 and not has_errors:\n logger.info(\"No new data added - all results may already exist in database\")\n else:\n logger.info(f\"✅ Sheets + Apify: {combined_added} new results to InstantDB (sheets={total_added}, apify={apify_added})\")\n\n if combined_added > 0:\n background_tasks.add_task(_post_scrape_accuracy_and_drl, request.game_type)\n\n return response\n\n\n@app.post(\"/api/cron/ingest-sheets\")\nasync def cron_ingest_sheets(\n background_tasks: BackgroundTasks,\n x_scrape_cron_secret: Optional[str] = Header(None, alias=\"X-Scrape-Cron-Secret\"),\n):\n \"\"\"\n Secured entry for Google Cloud Scheduler (or any cron) to update InstantDB from Google Sheets\n when no user has the site open. Set CRON_SCRAPE_SECRET in Cloud Run env and the same value\n in the scheduler's HTTP target header.\n \"\"\"\n if not Config.CRON_SCRAPE_SECRET:\n raise HTTPException(\n status_code=503,\n detail=\"CRON_SCRAPE_SECRET is not configured. Add it in Cloud Run environment variables to enable this endpoint.\",\n )\n if not x_scrape_cron_secret or x_scrape_cron_secret != Config.CRON_SCRAPE_SECRET:\n raise HTTPException(status_code=401, detail=\"Invalid or missing X-Scrape-Cron-Secret header\")\n\n try:\n return await _execute_sheets_scrape(\n ScrapeRequest(game_type=None, full_sync=False),\n background_tasks,\n full_sync=None,\n )\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error in cron ingest: {e}\")\n import traceback\n logger.error(traceback.format_exc())\n raise HTTPException(status_code=500, detail=f\"Scraping failed: {str(e)}\")\n\n\n@app.post(\"/api/scrape\")\nasync def scrape_data(\n request: ScrapeRequest,\n background_tasks: BackgroundTasks,\n full_sync: Optional[bool] = Query(None, description=\"If true, full CSV reconcile and reset sheet ingest cursor (overrides body when true)\"),\n):\n \"\"\"Trigger data scraping from Google Sheets. Can scrape all games or a specific game.\"\"\"\n try:\n return await _execute_sheets_scrape(request, background_tasks, full_sync)\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"Error in scrape endpoint: {e}\")\n import traceback\n logger.error(traceback.format_exc())\n raise HTTPException(\n status_code=500,\n detail=f\"Scraping failed: {str(e)}\"\n )\n\n\n@app.post(\"/api/ingest/apify\")\nasync def ingest_apify_dataset(request: ApifyIngestRequest):\n \"\"\"Poll Apify run dataset, normalize rows, dedupe, write to InstantDB (additive to Sheets).\"\"\"\n if not Config.APIFY_API_TOKEN:\n raise HTTPException(status_code=503, detail=\"APIFY_API_TOKEN is not configured\")\n if request.game_type and request.game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game_type\")\n try:\n result = ingest_apify_run(request.run_id, default_game_type=request.game_type)\n except ValueError as e:\n raise HTTPException(status_code=400, detail=str(e))\n except Exception as e:\n logger.exception(\"Apify ingest failed\")\n raise HTTPException(status_code=500, detail=str(e))\n\n if result.get(\"total_added\", 0) > 0:\n try:\n await auto_calculate_accuracy_for_new_results(\n request.game_type,\n results_limit=Config.ACCURACY_POST_SCRAPE_RESULTS_LIMIT,\n predictions_limit=Config.ACCURACY_POST_SCRAPE_PREDICTIONS_LIMIT,\n )\n except Exception as e:\n logger.warning(\"Auto-calculate after Apify ingest (non-critical): %s\", e)\n\n return {\"success\": True, **result, \"timestamp\": datetime.now().isoformat()}\n\n\n@app.post(\"/api/webhooks/apify\")\nasync def apify_webhook(\n req: Request,\n x_apify_secret: Optional[str] = Header(None, alias=\"X-Webhook-Secret\"),\n):\n \"\"\"Optional webhook: same ingest as POST /api/ingest/apify when Apify notifies a finished run.\"\"\"\n if Config.APIFY_WEBHOOK_SECRET:\n if x_apify_secret != Config.APIFY_WEBHOOK_SECRET:\n raise HTTPException(status_code=401, detail=\"Invalid webhook secret\")\n try:\n body = await req.json()\n except Exception:\n raise HTTPException(status_code=400, detail=\"Expected JSON body\")\n\n run_id = body.get(\"runId\") or body.get(\"run_id\") or body.get(\"resource\", {}).get(\"defaultRunId\")\n if not run_id:\n raise HTTPException(status_code=400, detail=\"runId not found in payload\")\n\n game_type = body.get(\"game_type\")\n if game_type and game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game_type\")\n\n if not Config.APIFY_API_TOKEN:\n raise HTTPException(status_code=503, detail=\"APIFY_API_TOKEN is not configured\")\n\n try:\n result = ingest_apify_run(str(run_id), default_game_type=game_type)\n except Exception as e:\n logger.exception(\"Apify webhook ingest failed\")\n raise HTTPException(status_code=500, detail=str(e))\n\n if result.get(\"total_added\", 0) > 0:\n try:\n await auto_calculate_accuracy_for_new_results(\n game_type,\n results_limit=Config.ACCURACY_POST_SCRAPE_RESULTS_LIMIT,\n predictions_limit=Config.ACCURACY_POST_SCRAPE_PREDICTIONS_LIMIT,\n )\n except Exception as e:\n logger.warning(\"Auto-calculate after Apify webhook (non-critical): %s\", e)\n\n return {\"success\": True, **result}\n\n\ndef _emit_predict_stream(stream_queue: Optional[queue.Queue], obj: Dict[str, Any]) -> None:\n if stream_queue is not None:\n stream_queue.put(json.dumps(obj, default=str) + \"\\n\")\n\n\ndef _prediction_pipeline(\n game_type: str,\n include_council: bool,\n stream_queue: Optional[queue.Queue] = None,\n) -> Optional[Dict[str, Any]]:\n \"\"\"Run six core models + Miro, optionally pushing NDJSON events to stream_queue.\"\"\"\n import time\n\n target_draw_date = date.today().isoformat()\n game_name = Config.GAMES[game_type]['name']\n\n print(\"=\" * 80)\n print(f\"STARTING PREDICTION GENERATION FOR {game_name}\")\n print(f\" Game Type: {game_type}\")\n print(f\" Target Date: {target_draw_date}\")\n print(\"=\" * 80)\n logger.info(\"=\" * 80)\n logger.info(f\"🎯 STARTING PREDICTION GENERATION FOR {game_name}\")\n logger.info(f\" Game Type: {game_type}\")\n logger.info(f\" Target Date: {target_draw_date}\")\n logger.info(\"=\" * 80)\n\n _emit_predict_stream(\n stream_queue,\n {\n \"event\": \"start\",\n \"game_type\": game_type,\n \"game_name\": game_name,\n \"target_draw_date\": target_draw_date,\n },\n )\n\n predictions: Dict[str, Any] = {}\n model_types = {\n 'XGBoost': xgboost_model,\n 'DecisionTree': decision_tree_model,\n 'MarkovChain': markov_chain_model,\n 'AnomalyDetection': anomaly_detection_model,\n 'NashHotFilter': nash_hot_filter_model,\n 'DRL': drl_agent,\n }\n\n all_prev_rows = instantdb.get_predictions(game_type, limit=120)\n\n def _run_model(name: str, inst: Any) -> Tuple[str, List[int], float]:\n start_time = time.time()\n timeout_seconds = 120 if name == 'DRL' else 60\n with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:\n future = executor.submit(inst.predict, game_type)\n try:\n predicted_numbers = future.result(timeout=timeout_seconds)\n except concurrent.futures.TimeoutError:\n raise TimeoutError(f\"{name} took longer than {timeout_seconds} seconds\")\n elapsed = time.time() - start_time\n return name, predicted_numbers, elapsed\n\n total_models = len(model_types)\n parallel = bool(getattr(Config, 'PREDICT_PARALLEL', True))\n max_workers = min(len(model_types), int(getattr(Config, 'PREDICT_MAX_WORKERS', 6)))\n\n def _persist_core(model_name: str, predicted_numbers: List[int], elapsed: float, idx_log: int) -> None:\n print(f\"\\n[{idx_log}/{total_models}] Training/Predicting with {model_name}...\")\n print(f\" ✅ {model_name} completed in {elapsed:.2f}s\")\n print(f\" 📊 Predicted numbers: {predicted_numbers}\")\n logger.info(f\"\\n[{idx_log}/{total_models}] 🤖 Training/Predicting with {model_name}...\")\n logger.info(f\" ✅ {model_name} completed in {elapsed:.2f}s\")\n logger.info(f\" 📊 Predicted numbers: {predicted_numbers}\")\n\n prev_preds = _prev_preds_lists_from_rows(all_prev_rows, model_name)\n print(f\" 💾 Creating NEW {model_name} prediction in database...\")\n logger.info(f\" 💾 Creating NEW {model_name} prediction in database (target_date: {target_draw_date})...\")\n prediction_data = {\n 'target_draw_date': target_draw_date,\n 'model_type': model_name,\n 'predicted_number_1': predicted_numbers[0],\n 'predicted_number_2': predicted_numbers[1],\n 'predicted_number_3': predicted_numbers[2],\n 'predicted_number_4': predicted_numbers[3],\n 'predicted_number_5': predicted_numbers[4],\n 'predicted_number_6': predicted_numbers[5],\n 'previous_prediction_1': prev_preds[0] if len(prev_preds) > 0 else None,\n 'previous_prediction_2': prev_preds[1] if len(prev_preds) > 1 else None,\n 'previous_prediction_3': prev_preds[2] if len(prev_preds) > 2 else None,\n 'previous_prediction_4': prev_preds[3] if len(prev_preds) > 3 else None,\n 'previous_prediction_5': prev_preds[4] if len(prev_preds) > 4 else None,\n 'created_at': datetime.now().isoformat(),\n }\n new_prediction = instantdb.create_prediction(game_type, prediction_data)\n prediction_id = new_prediction.get('id', 'unknown')\n print(f\" ✅ Created NEW prediction record with ID: {prediction_id}\")\n logger.info(f\" ✅ Created NEW prediction record with ID: {prediction_id} (old predictions preserved)\")\n predictions[model_name] = {\n 'numbers': predicted_numbers,\n 'previous_predictions': prev_preds[:5],\n 'prediction_id': new_prediction.get('id'),\n }\n _emit_predict_stream(\n stream_queue,\n {\n \"event\": \"model\",\n \"model\": model_name,\n \"elapsed_sec\": round(elapsed, 3),\n \"predictions\": {model_name: predictions[model_name]},\n },\n )\n\n def _fail_core(model_name: str, err: str, idx_log: int) -> None:\n print(f\"\\n[{idx_log}/{total_models}] Training/Predicting with {model_name}...\")\n print(f\" ❌ {model_name} FAILED: {err}\")\n logger.error(f\"\\n[{idx_log}/{total_models}] 🤖 {model_name} FAILED: {err}\")\n predictions[model_name] = {'error': err}\n _emit_predict_stream(\n stream_queue,\n {\"event\": \"model\", \"model\": model_name, \"predictions\": {model_name: {'error': err}}},\n )\n\n completion_counter = [0]\n\n def _next_idx() -> int:\n completion_counter[0] += 1\n return completion_counter[0]\n\n if parallel and max_workers > 1:\n with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:\n future_to_name = {\n executor.submit(_run_model, model_name, model_instance): model_name\n for model_name, model_instance in model_types.items()\n }\n for future in concurrent.futures.as_completed(future_to_name):\n model_name = future_to_name[future]\n idx_log = _next_idx()\n try:\n name, nums, elapsed = future.result()\n except Exception as e:\n _fail_core(model_name, str(e), idx_log)\n continue\n _persist_core(name, nums, elapsed, idx_log)\n else:\n for model_name, model_instance in model_types.items():\n idx_log = _next_idx()\n try:\n name, nums, elapsed = _run_model(model_name, model_instance)\n except Exception as e:\n _fail_core(model_name, str(e), idx_log)\n continue\n _persist_core(name, nums, elapsed, idx_log)\n\n if getattr(Config, \"MIRO_STRATEGY_ENABLED\", True):\n miro_label = \"Miro\"\n if Config.LLM_API_KEY:\n logger.info(\"Running Miro strategy (LLM multi-agent)…\")\n try:\n with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:\n fut = pool.submit(run_miro_strategy_predict, game_type, predictions)\n miro_numbers = fut.result(timeout=180)\n prev_preds = _prev_preds_lists_from_rows(all_prev_rows, miro_label)\n prediction_data = {\n \"target_draw_date\": target_draw_date,\n \"model_type\": miro_label,\n \"predicted_number_1\": miro_numbers[0],\n \"predicted_number_2\": miro_numbers[1],\n \"predicted_number_3\": miro_numbers[2],\n \"predicted_number_4\": miro_numbers[3],\n \"predicted_number_5\": miro_numbers[4],\n \"predicted_number_6\": miro_numbers[5],\n \"previous_prediction_1\": prev_preds[0] if len(prev_preds) > 0 else None,\n \"previous_prediction_2\": prev_preds[1] if len(prev_preds) > 1 else None,\n \"previous_prediction_3\": prev_preds[2] if len(prev_preds) > 2 else None,\n \"previous_prediction_4\": prev_preds[3] if len(prev_preds) > 3 else None,\n \"previous_prediction_5\": prev_preds[4] if len(prev_preds) > 4 else None,\n \"created_at\": datetime.now().isoformat(),\n }\n new_prediction = instantdb.create_prediction(game_type, prediction_data)\n predictions[miro_label] = {\n \"numbers\": miro_numbers,\n \"previous_predictions\": prev_preds[:5],\n \"prediction_id\": new_prediction.get(\"id\"),\n }\n logger.info(\"Miro prediction stored: %s\", miro_numbers)\n _emit_predict_stream(\n stream_queue,\n {\"event\": \"model\", \"model\": miro_label, \"predictions\": {miro_label: predictions[miro_label]}},\n )\n except concurrent.futures.TimeoutError:\n logger.error(\"Miro strategy timed out after 180s\")\n predictions[miro_label] = {\"error\": \"Miro strategy timed out (180s)\"}\n _emit_predict_stream(\n stream_queue,\n {\"event\": \"model\", \"model\": miro_label, \"predictions\": {miro_label: predictions[miro_label]}},\n )\n except Exception as e:\n logger.exception(\"Miro strategy failed\")\n predictions[miro_label] = {\"error\": str(e)}\n _emit_predict_stream(\n stream_queue,\n {\"event\": \"model\", \"model\": miro_label, \"predictions\": {miro_label: predictions[miro_label]}},\n )\n else:\n predictions[miro_label] = {\n \"error\": \"LLM_API_KEY is not configured (required for Miro strategy)\",\n }\n _emit_predict_stream(\n stream_queue,\n {\"event\": \"model\", \"model\": miro_label, \"predictions\": {miro_label: predictions[miro_label]}},\n )\n else:\n predictions[\"Miro\"] = {\n \"error\": \"Miro strategy is disabled (MIRO_STRATEGY_ENABLED=false)\",\n }\n _emit_predict_stream(\n stream_queue,\n {\"event\": \"model\", \"model\": \"Miro\", \"predictions\": {\"Miro\": predictions[\"Miro\"]}},\n )\n\n n_cards = len(predictions)\n print(\"\\n\" + \"=\" * 80)\n print(f\"PREDICTION GENERATION COMPLETE!\")\n print(f\" Successful: {sum(1 for p in predictions.values() if 'error' not in p)}/{n_cards}\")\n print(f\" Failed: {sum(1 for p in predictions.values() if 'error' in p)}/{n_cards}\")\n print(\"=\" * 80)\n logger.info(\"\\n\" + \"=\" * 80)\n logger.info(f\"🎉 PREDICTION GENERATION COMPLETE!\")\n logger.info(f\" Successful: {sum(1 for p in predictions.values() if 'error' not in p)}/{n_cards}\")\n logger.info(f\" Failed: {sum(1 for p in predictions.values() if 'error' in p)}/{n_cards}\")\n logger.info(\"=\" * 80)\n\n try:\n def run_auto_calculate():\n import asyncio as aio\n try:\n aio.run(auto_calculate_accuracy_for_new_results(game_type))\n except Exception as e:\n logger.warning(f\"Background auto-calculation failed: {e}\")\n\n thread = threading.Thread(target=run_auto_calculate)\n thread.daemon = True\n thread.start()\n logger.info(f\"✅ Triggered background auto-calculation of accuracy for {game_type}\")\n except Exception as e:\n logger.warning(f\"Failed to trigger auto-calculation after prediction generation: {e}\")\n\n council_report = None\n if include_council and Config.LLM_API_KEY and getattr(Config, \"LLM_COUNCIL_ENABLED\", True):\n try:\n council_report = run_council_report(game_type, predictions, \"\")\n except AuthenticationError as e:\n logger.warning(\"Council report auth failed (non-critical): %s\", e)\n council_report = {\"error\": \"LLM API key rejected; check LLM_API_KEY in .env\"}\n except Exception as e:\n logger.warning(\"Council report failed (non-critical): %s\", e)\n council_report = {\"error\": str(e)}\n\n ts = datetime.now().isoformat()\n if stream_queue is not None:\n _emit_predict_stream(\n stream_queue,\n {\n \"event\": \"done\",\n \"success\": True,\n \"game_type\": game_type,\n \"target_draw_date\": target_draw_date,\n \"predictions\": predictions,\n \"timestamp\": ts,\n \"council_report\": council_report,\n },\n )\n return None\n\n response_payload: Dict[str, Any] = {\n 'success': True,\n 'game_type': game_type,\n 'target_draw_date': target_draw_date,\n 'predictions': predictions,\n 'timestamp': ts,\n }\n if council_report is not None:\n response_payload['council_report'] = council_report\n return response_payload\n\n\n@app.post(\"/api/predict/{game_type}\")\nasync def generate_predictions(\n game_type: str = Path(..., description=\"Game type identifier\"),\n include_council: bool = Query(False, description=\"Run LLM council after models (requires LLM_API_KEY)\"),\n stream: bool = Query(False, description=\"Stream NDJSON (application/x-ndjson) as each model finishes\"),\n):\n \"\"\"Get predictions from six core ML models, then optional Miro (LLM).\"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n\n if stream:\n q: queue.Queue = queue.Queue()\n\n def worker():\n try:\n _prediction_pipeline(game_type, include_council, stream_queue=q)\n except Exception as e:\n logger.exception(\"Streaming prediction pipeline failed\")\n q.put(json.dumps({\"event\": \"error\", \"detail\": str(e)}) + \"\\n\")\n finally:\n q.put(None)\n\n threading.Thread(target=worker, daemon=True).start()\n\n async def ndjson_iter():\n while True:\n line = await asyncio.to_thread(q.get)\n if line is None:\n break\n yield line\n\n return StreamingResponse(ndjson_iter(), media_type=\"application/x-ndjson\")\n\n try:\n out = _prediction_pipeline(game_type, include_council, stream_queue=None)\n return out\n except HTTPException:\n raise\n except Exception as e:\n logger.error(f\"❌ PREDICTION GENERATION FAILED: {str(e)}\")\n raise HTTPException(status_code=500, detail=str(e))\n\n\n@app.post(\"/api/predict/{game_type}/council-report\")\nasync def council_report(game_type: str, body: CouncilReportRequest = Body(...)):\n \"\"\"LLM council summary + six-agent JSON (advisory).\"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n if not Config.LLM_API_KEY:\n raise HTTPException(status_code=503, detail=\"LLM_API_KEY is not configured\")\n\n preds = body.predictions\n if body.use_latest or not preds:\n preds = load_latest_predictions_for_council(game_type)\n if not preds:\n raise HTTPException(status_code=400, detail=\"No predictions found; run predict first or pass predictions\")\n\n mem = build_memory_context_for_council(body.user_key)\n try:\n report = run_council_report(game_type, preds, mem)\n except (AuthenticationError, RateLimitError, APIConnectionError) as e:\n logger.warning(\"council-report LLM error: %s\", e)\n raise _http_exception_from_llm_error(e)\n except Exception as e:\n logger.exception(\"council-report failed\")\n raise HTTPException(status_code=500, detail=str(e))\n\n if body.user_key:\n try:\n summary_text = json.dumps(report.get(\"summary\", {}), ensure_ascii=False)[:8000]\n upsert_user_memory(body.user_key, last_summary=summary_text)\n except Exception as e:\n logger.debug(\"Optional user_memory upsert failed: %s\", e)\n\n return {\"success\": True, \"report\": report}\n\n\n@app.get(\"/api/memory/user/{user_key:path}\")\nasync def get_user_memory(user_key: str):\n rec = get_user_memory_record(user_key)\n return {\"user_key\": user_key, \"record\": rec}\n\n\n@app.post(\"/api/memory/user\")\nasync def post_user_memory(body: MemoryUpsertRequest):\n try:\n out = upsert_user_memory(\n body.user_key,\n pinned_games=body.pinned_games,\n preferences=body.preferences,\n last_summary=body.last_summary,\n )\n return {\"success\": True, **out}\n except Exception as e:\n logger.exception(\"user_memory upsert failed\")\n raise HTTPException(status_code=500, detail=str(e))\n\n\n@app.get(\"/api/graphs/{game_type}/cooccurrence\")\nasync def graph_cooccurrence(\n game_type: str,\n limit_draws: Optional[int] = Query(\n None,\n ge=50,\n le=5000,\n description=\"How many recent draws to aggregate (default from GRAPH_AGG_LIMIT_DRAWS)\",\n ),\n):\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n try:\n return cooccurrence_edges(game_type, limit_draws=limit_draws)\n except Exception as e:\n logger.exception(\"cooccurrence graph failed\")\n raise HTTPException(status_code=500, detail=str(e))\n\n\n@app.get(\"/api/graphs/{game_type}/markov-edges\")\nasync def graph_markov_edges(\n game_type: str,\n limit_draws: Optional[int] = Query(\n None,\n ge=50,\n le=5000,\n description=\"How many recent draws to aggregate (default from GRAPH_AGG_LIMIT_DRAWS)\",\n ),\n):\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n try:\n return markov_ball_transitions(game_type, limit_draws=limit_draws)\n except Exception as e:\n logger.exception(\"markov-edges graph failed\")\n raise HTTPException(status_code=500, detail=str(e))\n\n\n@app.post(\"/api/graphs/{game_type}/sankey\")\nasync def graph_sankey(game_type: str, body: SankeyRequest):\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n try:\n return sankey_hot_model_votes(game_type, body.predictions)\n except Exception as e:\n logger.exception(\"sankey graph failed\")\n raise HTTPException(status_code=500, detail=str(e))\n\n\n@app.get(\"/api/predictions/{game_type}\")\nasync def get_predictions(\n game_type: str = Path(..., description=\"Game type identifier\"),\n limit: int = Query(10, ge=1, le=100, description=\"Number of predictions to return\")\n):\n \"\"\"Get stored predictions for a game.\"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n \n predictions = instantdb.get_predictions(game_type, limit=limit)\n \n predictions_data = []\n for pred in predictions:\n prev_preds = []\n for prev_pred_key in ['previous_prediction_1', 'previous_prediction_2',\n 'previous_prediction_3', 'previous_prediction_4',\n 'previous_prediction_5']:\n prev_pred = pred.get(prev_pred_key)\n if prev_pred:\n try:\n # Parse JSON if stored as string\n if isinstance(prev_pred, str):\n prev_preds.append(json.loads(prev_pred))\n else:\n prev_preds.append(prev_pred)\n except:\n pass\n \n predictions_data.append({\n 'id': pred.get('id'),\n 'model_type': pred.get('model_type'),\n 'target_draw_date': pred.get('target_draw_date'),\n 'numbers': [\n pred.get('predicted_number_1'), pred.get('predicted_number_2'), pred.get('predicted_number_3'),\n pred.get('predicted_number_4'), pred.get('predicted_number_5'), pred.get('predicted_number_6')\n ],\n 'previous_predictions': prev_preds,\n 'created_at': pred.get('created_at')\n })\n \n return {\n 'predictions': predictions_data\n }\n\n@app.post(\"/api/predictions/{prediction_id}/calculate-accuracy\")\nasync def calculate_accuracy(\n prediction_id: str = Path(..., description=\"Prediction ID\"),\n request: CalculateAccuracyRequest = Body(...)\n):\n \"\"\"Calculate and store error distance when actual results are available.\"\"\"\n result_id = request.result_id\n game_type = request.game_type\n \n if not result_id or not game_type:\n raise HTTPException(status_code=400, detail=\"Missing result_id or game_type\")\n \n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n \n try:\n # Get prediction and result from InstantDB\n predictions = instantdb.get_predictions(game_type, limit=1000)\n prediction = next((p for p in predictions if p.get('id') == prediction_id), None)\n \n results = instantdb.get_results(game_type, limit=1000)\n result = next((r for r in results if r.get('id') == result_id), None)\n \n if not prediction or not result:\n raise HTTPException(status_code=404, detail=\"Prediction or result not found\")\n \n predicted = [\n prediction.get('predicted_number_1'), prediction.get('predicted_number_2'),\n prediction.get('predicted_number_3'), prediction.get('predicted_number_4'),\n prediction.get('predicted_number_5'), prediction.get('predicted_number_6')\n ]\n \n actual = [\n result.get('number_1'), result.get('number_2'), result.get('number_3'),\n result.get('number_4'), result.get('number_5'), result.get('number_6')\n ]\n \n # Calculate metrics\n metrics = calculate_all_metrics(predicted, actual)\n # Convert numpy types to native Python types for JSON serialization\n metrics_clean = convert_to_python_types(metrics)\n \n # Store accuracy in InstantDB\n accuracy_data = {\n 'prediction_id': prediction_id,\n 'result_id': result_id,\n 'error_distance': float(metrics_clean.get('error_distance')),\n 'numbers_matched': int(metrics_clean.get('numbers_matched')),\n 'distance_metrics': metrics_clean,\n 'calculated_at': datetime.now().isoformat()\n }\n \n instantdb.create_prediction_accuracy(game_type, accuracy_data)\n \n # Trigger DRL learning from stored accuracy records if this is a DRL prediction\n if prediction.get('model_type') == 'DRL':\n try:\n # Get recent accuracy records for this game (DRL predictions only)\n accuracy_records = instantdb.get_prediction_accuracy(game_type)\n \n # Filter for DRL predictions\n predictions_all = instantdb.get_predictions(game_type, limit=1000)\n drl_prediction_ids = {p.get('id') for p in predictions_all if p.get('model_type') == 'DRL'}\n drl_accuracy = [acc for acc in accuracy_records if acc.get('prediction_id') in drl_prediction_ids]\n \n # Update DRL agent with new feedback\n if len(drl_accuracy) >= 5: # Only update if we have enough data\n drl_agent.learn_from_accuracy_records(game_type, drl_accuracy, instantdb)\n logger.info(f\"DRL agent updated with accuracy feedback for {game_type}\")\n except Exception as e:\n logger.warning(f\"Failed to update DRL with accuracy (non-critical): {e}\")\n \n return {\n 'success': True,\n 'metrics': metrics\n }\n \n except HTTPException:\n raise\n except Exception as e:\n raise HTTPException(status_code=500, detail=str(e))\n\n@app.get(\"/api/predictions/{game_type}/accuracy\")\nasync def get_prediction_accuracy(\n game_type: str = Path(..., description=\"Game type identifier\"),\n limit: int = Query(100, ge=1, le=1000, description=\"Number of records to return\")\n):\n \"\"\"Get prediction accuracy metrics and error distance analysis.\"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n \n accuracy_records = instantdb.get_prediction_accuracy(game_type)\n \n # Limit results\n accuracy_records = accuracy_records[:limit]\n \n accuracy_data = []\n for record in accuracy_records:\n distance_metrics = record.get('distance_metrics')\n if isinstance(distance_metrics, str):\n try:\n distance_metrics = json.loads(distance_metrics)\n except:\n distance_metrics = {}\n \n accuracy_data.append({\n 'id': record.get('id'),\n 'prediction_id': record.get('prediction_id'),\n 'result_id': record.get('result_id'),\n 'error_distance': record.get('error_distance'),\n 'numbers_matched': record.get('numbers_matched'),\n 'distance_metrics': distance_metrics,\n 'calculated_at': record.get('calculated_at')\n })\n \n return {\n 'accuracy_records': accuracy_data\n }\n\n@app.get(\"/api/stats/{game_type}\")\nasync def get_statistics(game_type: str = Path(..., description=\"Game type identifier\")):\n \"\"\"Get frequency statistics for a game.\"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n\n def _compute() -> Dict[str, Any]:\n cap = int(getattr(Config, \"STATS_RESULTS_LIMIT\", 5000))\n all_results = instantdb.get_results(\n game_type, limit=cap, offset=0, order_by='draw_date.desc'\n )\n hot_numbers = get_hot_numbers(game_type, top_n=20, preloaded_results=all_results)\n cold_numbers = get_cold_numbers(game_type, bottom_n=20, preloaded_results=all_results)\n overdue_numbers = get_overdue_numbers(game_type, preloaded_results=all_results)\n total_draws = len(all_results)\n\n date_range = None\n if all_results:\n dates = [r.get('draw_date') for r in all_results if r.get('draw_date')]\n if dates:\n date_range = {\n 'start': min(dates),\n 'end': max(dates)\n }\n\n jackpots = [float(r.get('jackpot')) for r in all_results if r.get('jackpot')]\n avg_jackpot = None\n if jackpots:\n avg_jackpot = sum(jackpots) / len(jackpots)\n\n return {\n 'hot_numbers': [{'number': num, 'frequency': freq} for num, freq in hot_numbers],\n 'cold_numbers': [{'number': num, 'frequency': freq} for num, freq in cold_numbers],\n 'overdue_numbers': [{'number': num, 'days_since': days} for num, days in overdue_numbers[:20]],\n 'total_draws': total_draws,\n 'date_range': date_range,\n 'average_jackpot': avg_jackpot\n }\n\n try:\n ttl = float(getattr(Config, \"STATS_API_CACHE_TTL_SEC\", 90))\n return get_or_set(_stats_api_cache, ttl, game_type, _compute)\n except Exception as e:\n raise HTTPException(status_code=500, detail=str(e))\n\n@app.get(\"/api/stats/{game_type}/gaussian\")\nasync def get_gaussian_distribution(game_type: str = Path(..., description=\"Game type identifier\")):\n \"\"\"Get Gaussian distribution analysis of winning numbers.\"\"\"\n if game_type not in Config.GAMES:\n raise HTTPException(status_code=400, detail=\"Invalid game type\")\n\n def _compute() -> Dict[str, Any]:\n cap = int(getattr(Config, \"STATS_RESULTS_LIMIT\", 5000))\n all_results = instantdb.get_results(game_type, limit=cap, offset=0, order_by='draw_date.asc')\n\n distribution_data = []\n for result in all_results:\n numbers = [\n result.get('number_1'), result.get('number_2'), result.get('number_3'),\n result.get('number_4'), result.get('number_5'), result.get('number_6')\n ]\n\n numbers = [n for n in numbers if n is not None]\n\n if len(numbers) == 6:\n numbers_sum = sum(numbers)\n numbers_product = 1\n for num in numbers:\n numbers_product *= num\n\n distribution_data.append({\n 'sum': numbers_sum,\n 'product': numbers_product,\n 'draw_date': result.get('draw_date'),\n 'numbers': numbers,\n 'winners': result.get('winners', 0),\n 'jackpot': result.get('jackpot', 0)\n })\n \n # Calculate statistics for Gaussian curve\n if distribution_data:\n sums = [d['sum'] for d in distribution_data]\n products = [d['product'] for d in distribution_data]\n \n import numpy as np\n \n # Calculate mean and std for sum\n sum_mean = np.mean(sums)\n sum_std = np.std(sums)\n \n # Calculate mean and std for product (use log scale due to large values)\n log_products = [np.log(p) if p > 0 else 0 for p in products]\n product_mean = np.mean(products)\n product_std = np.std(products)\n log_product_mean = np.mean(log_products)\n log_product_std = np.std(log_products)\n \n return {\n 'distribution_data': distribution_data,\n 'statistics': {\n 'sum': {\n 'mean': float(sum_mean),\n 'std': float(sum_std),\n 'min': min(sums),\n 'max': max(sums),\n 'count': len(sums)\n },\n 'product': {\n 'mean': float(product_mean),\n 'std': float(product_std),\n 'min': min(products),\n 'max': max(products),\n 'log_mean': float(log_product_mean),\n 'log_std': float(log_product_std),\n 'count': len(products)\n }\n }\n }\n \n return {\n 'distribution_data': [],\n 'statistics': None\n }\n\n try:\n ttl = float(getattr(Config, \"STATS_API_CACHE_TTL_SEC\", 90))\n return get_or_set(_gaussian_api_cache, ttl, game_type, _compute)\n except Exception as e:\n logger.error(f\"Error calculating Gaussian distribution: {e}\")\n raise HTTPException(status_code=500, detail=str(e))\n\n@app.get(\"/api/health\")\nasync def health_check():\n \"\"\"Health check endpoint.\"\"\"\n return {\n 'status': 'healthy',\n 'database': 'InstantDB',\n 'timestamp': datetime.now().isoformat()\n }\n\nif __name__ == '__main__':\n import uvicorn\n uvicorn.run(\n \"app:app\",\n host=\"0.0.0.0\",\n port=5000,\n reload=Config.DEBUG\n )\n"}}}} |