{"repo": "adityaakashbhu23-svg/cv-toposheet-maps", "files": {"app.py": {"475b54e9f2dbd1f8ba21959ecc488867fefb292e": {"t": 1785674392.2617524, "n": 129487, "h": "5f98c327ddfff7c6", "c": "# app.py – CV-Toposheet Web Interface\n#\n# A simple Flask cover page: upload a map → pipeline runs → view results.\n#\n\nimport sys\nimport time\nimport threading\nimport queue\nimport json\nimport os\nimport re\nimport uuid\nimport shutil\nimport sqlite3\nimport importlib\nimport subprocess\nfrom pathlib import Path\nfrom urllib.parse import quote as url_quote, urlparse\n\nfrom flask import Flask, request, redirect, url_for, send_file, Response, stream_with_context, jsonify\nfrom werkzeug.utils import secure_filename\n\n# In a frozen PyInstaller EXE, __file__ is inside _internal/ (read-only bundle dir).\n# User-writable data (maps, results, .env, JSON credentials) must go next to the EXE.\nif getattr(sys, 'frozen', False):\n BASE_DIR = Path(sys.executable).parent # install dir e.g. C:\\...\\CV-Toposheet\\\nelse:\n BASE_DIR = Path(__file__).parent # project root (dev mode)\n\n# MSIX (WindowsApps) and macOS .app bundles are read-only — use a writable user dir\n# for ALL data that the app writes at runtime (maps, results, logs, flags, .env).\nimport platform as _platform\nif getattr(sys, 'frozen', False):\n if _platform.system() == 'Darwin':\n _USER_DATA_DIR = Path.home() / 'Library' / 'Application Support' / 'CVToposheet'\n else:\n # Windows: %LOCALAPPDATA%\\CVToposheet\\ (writable even from MSIX sandbox)\n _USER_DATA_DIR = Path(os.environ.get('LOCALAPPDATA', Path.home())) / 'CVToposheet'\n _USER_DATA_DIR.mkdir(parents=True, exist_ok=True)\nelse:\n _USER_DATA_DIR = BASE_DIR\n\nRESULTS_DIR = _USER_DATA_DIR / 'results'\nMAPS_DIR = _USER_DATA_DIR / 'maps'\nLOGS_DIR = _USER_DATA_DIR / 'logs'\n\nFIRST_RUN_FLAG = _USER_DATA_DIR / '.welcome_done'\nENV_FILE = _USER_DATA_DIR / '.env'\nAI_REPORTS_FILE = LOGS_DIR / 'ai_content_reports.jsonl'\n\n# Load saved API keys / settings from the writable user data dir on every launch.\nfrom dotenv import load_dotenv as _load_dotenv\n_load_dotenv(ENV_FILE, override=False) # override=False: don't clobber env vars already set\n\n# ── Fresh-reinstall detection (MSIX / Mac .app / EXE) ────────────────────────\n# On every (re)install the app binary is brand-new, so its ctime is later than\n# any flag written during a previous install. If we detect this, delete the\n# flag so the welcome page shows once more for that fresh install.\ndef _reset_flag_if_reinstalled() -> None:\n if not getattr(sys, 'frozen', False):\n return # dev mode — never auto-reset\n if not FIRST_RUN_FLAG.exists():\n return # flag not written yet — nothing to reset\n try:\n exe_ctime = Path(sys.executable).stat().st_ctime\n flag_mtime = FIRST_RUN_FLAG.stat().st_mtime\n if exe_ctime > flag_mtime:\n # Executable is newer than the flag → fresh reinstall\n FIRST_RUN_FLAG.unlink(missing_ok=True)\n except Exception:\n pass # never break startup over this\n\n_reset_flag_if_reinstalled()\n# ─────────────────────────────────────────────────────────────────────────────\n\nSUPPORT_ISSUES_URL = 'https://github.com/adityaakashbhu23-svg/CV-Toposheet-Maps/issues/new'\nSUPPORT_EMAIL = 'cvtoposheet@outlook.com'\n\nALLOWED_EXT = {'.jpg', '.jpeg', '.png', '.tif', '.tiff'}\n\n_process_lock = threading.Lock()\n_current_proc = None # currently running subprocess (for kill support)\n\n_MODEL_ENVS = {\n 'best': {'LLM_PROVIDER': 'vertex', 'OCR_ENGINE': 'gcv'},\n 'fast': {'LLM_PROVIDER': 'groq', 'OCR_ENGINE': 'gcv'},\n 'ensemble': {'LLM_PROVIDER': 'vertex', 'OCR_ENGINE': 'gcv', 'ENSEMBLE_MODE': 'true'},\n 'openai': {'LLM_PROVIDER': 'openai', 'OCR_ENGINE': 'gcv'},\n 'claude': {'LLM_PROVIDER': 'claude', 'OCR_ENGINE': 'gcv'},\n 'grok': {'LLM_PROVIDER': 'grok', 'OCR_ENGINE': 'gcv'},\n 'gemini': {'LLM_PROVIDER': 'gemini', 'OCR_ENGINE': 'gcv'},\n 'openrouter': {'LLM_PROVIDER': 'openrouter', 'OCR_ENGINE': 'gcv'},\n 'offline': {'LLM_PROVIDER': 'groq', 'OCR_ENGINE': 'easyocr', 'OCR_WORKERS': '1'},\n}\n\napp = Flask(__name__)\napp.config['MAX_CONTENT_LENGTH'] = 500 * 1024 * 1024 # 500 MB\n\n\n# ── Helpers ───────────────────────────────────────────────────────────────────\ndef _feature_count() -> int:\n db = RESULTS_DIR / 'toposheet.db'\n if not db.exists():\n return 0\n try:\n with sqlite3.connect(str(db)) as con:\n return con.execute('SELECT COUNT(*) FROM features').fetchone()[0]\n except Exception:\n return 0\n\n\ndef _map_count() -> int:\n if not MAPS_DIR.exists():\n return 0\n return sum(1 for _ in MAPS_DIR.glob('*') if _.is_file() and _.suffix.lower() in ALLOWED_EXT)\n\n\ndef _merge_session_to_global(session_id: str) -> None:\n \"\"\"Copy newly processed map(s) from a session DB into the global DB,\n then regenerate the global table_export.html so Map Database stays current.\"\"\"\n session_db = _USER_DATA_DIR / 'results' / session_id / 'toposheet.db'\n global_db = RESULTS_DIR / 'toposheet.db'\n if not session_db.exists():\n return\n try:\n from utils.db_utils import init_db\n if not global_db.exists():\n init_db(str(global_db))\n\n with sqlite3.connect(str(global_db)) as gcon:\n gcon.execute('ATTACH DATABASE ? AS sdb', (str(session_db),))\n maps_in_session = [r[0] for r in gcon.execute(\n 'SELECT DISTINCT map_name FROM sdb.features'\n ).fetchall()]\n inserted = 0\n for map_name in maps_in_session:\n already = gcon.execute(\n 'SELECT 1 FROM features WHERE map_name=? LIMIT 1', (map_name,)\n ).fetchone()\n if not already:\n cur = gcon.execute(\"\"\"\n INSERT INTO features\n (map_name, original_text, feature_name, feature_type,\n grid_reference, confidence, tile_x, tile_y,\n bbox_x1, bbox_y1, bbox_x2, bbox_y2,\n lat_min, lat_max, lon_min, lon_max,\n sheet_ref, district, survey_year, map_scale, verified)\n SELECT map_name, original_text, feature_name, feature_type,\n grid_reference, confidence, tile_x, tile_y,\n bbox_x1, bbox_y1, bbox_x2, bbox_y2,\n lat_min, lat_max, lon_min, lon_max,\n sheet_ref, district, survey_year, map_scale, verified\n FROM sdb.features WHERE map_name=?\n \"\"\", (map_name,))\n inserted += cur.rowcount\n gcon.commit()\n\n if inserted > 0:\n # Regenerate the global HTML so Map Database reflects the new map\n genv = os.environ.copy()\n genv['RESULTS_FOLDER'] = str(RESULTS_DIR)\n genv['PYTHONIOENCODING'] = 'utf-8'\n result = subprocess.run(\n [sys.executable, 'export_table.py'],\n cwd=str(BASE_DIR), env=genv,\n timeout=120, capture_output=True\n )\n if result.returncode != 0:\n print('[merge] export_table.py failed:', result.stderr.decode(errors='replace'))\n print(f'[merge] Added {inserted} features from session {session_id} to global DB.')\n else:\n print(f'[merge] Session {session_id} maps already in global DB — skipped.')\n except Exception as e:\n print(f'[merge] Warning: could not merge session to global DB: {e}')\n\n\ndef _write_env(env: dict):\n lines = []\n if ENV_FILE.exists():\n with open(ENV_FILE, 'r', encoding='utf-8') as f:\n lines = f.readlines()\n existing_keys = {}\n for i, line in enumerate(lines):\n m = re.match(r'^([A-Z_][A-Z0-9_]*)=', line)\n if m:\n existing_keys[m.group(1)] = i\n for k, v in env.items():\n if v:\n entry = f'{k}={v}\\n'\n if k in existing_keys:\n lines[existing_keys[k]] = entry\n else:\n lines.append(entry)\n else:\n # Empty value — remove the key from .env so it doesn't come back on restart\n if k in existing_keys:\n lines[existing_keys[k]] = None # mark for removal\n lines = [l for l in lines if l is not None]\n with open(ENV_FILE, 'w', encoding='utf-8') as f:\n f.writelines(lines)\n\n\n# ── Routes ────────────────────────────────────────────────────────────────────\n@app.route('/')\ndef index():\n if not FIRST_RUN_FLAG.exists():\n return redirect('/welcome')\n try:\n feat_count = _feature_count()\n except Exception:\n feat_count = 0\n try:\n map_count = _map_count()\n except Exception:\n map_count = 0\n return _landing_html(feat_count, map_count)\n\n\n@app.route('/welcome')\ndef welcome():\n return _WELCOME_TEMPLATE\n\n\n@app.route('/dismiss_welcome', methods=['POST'])\ndef dismiss_welcome():\n try:\n FIRST_RUN_FLAG.touch()\n except Exception as e:\n app.logger.error(\"Could not write welcome flag to %s: %s\", FIRST_RUN_FLAG, e)\n return redirect('/')\n\n\n@app.route('/upload_service_account', methods=['POST'])\ndef upload_service_account():\n f = request.files.get('json_file')\n target = request.form.get('target', 'service_account.json')\n # Only allow safe filenames — must end in .json, no path traversal\n if not f or not f.filename.endswith('.json'):\n return json.dumps({'ok': False, 'error': 'Not a JSON file'}), 400\n safe_name = Path(target).name # strip any path component\n if not safe_name.endswith('.json'):\n safe_name = 'service_account.json'\n dest = _USER_DATA_DIR / safe_name\n f.save(str(dest))\n # Update env so it takes effect immediately\n os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = str(dest)\n try:\n _write_env({'GOOGLE_APPLICATION_CREDENTIALS': safe_name})\n import config; importlib.reload(config)\n except Exception:\n pass\n return json.dumps({'ok': True, 'saved': safe_name})\n\n\ndef _session_feature_count(session_id: str) -> int:\n \"\"\"Return feature row count for a session DB (0 if missing/error).\"\"\"\n db = _USER_DATA_DIR / 'results' / session_id / 'toposheet.db'\n if not db.exists():\n return 0\n try:\n with sqlite3.connect(str(db)) as con:\n row = con.execute('SELECT COUNT(*) FROM features').fetchone()\n return int(row[0] or 0)\n except Exception:\n return 0\n\n\n@app.route('/save_env', methods=['POST'])\ndef save_env():\n ref = request.referrer or ''\n if not (ref.startswith('http://127.0.0.1:') or ref.startswith('http://localhost:')):\n return '', 403\n payload = request.get_json(force=True, silent=True) or {}\n # Normalize legacy/blank OpenRouter model to the new default.\n if 'OPENROUTER_MODEL' in payload:\n old_openrouter_default = 'meta-llama/llama-3.3-70b-instruct:free'\n or_model = str(payload.get('OPENROUTER_MODEL') or '').strip()\n if (not or_model) or (or_model.lower() == old_openrouter_default):\n payload['OPENROUTER_MODEL'] = 'openrouter/auto'\n for k, v in payload.items():\n if v:\n os.environ[k] = v\n elif k in os.environ:\n del os.environ[k]\n try:\n _write_env(payload) # pass all keys including empty — _write_env removes blanked keys\n except Exception:\n pass\n try:\n import config\n importlib.reload(config)\n except Exception:\n pass\n return json.dumps({'ok': True})\n\n\n@app.route('/report_ai_content', methods=['POST'])\ndef report_ai_content():\n try:\n ref = request.referrer or ''\n if ref:\n ref_host = (urlparse(ref).hostname or '').lower()\n req_host = (request.host.split(':', 1)[0] if request.host else '').lower()\n allowed_hosts = {'127.0.0.1', 'localhost', '0.0.0.0'}\n if ref_host and ref_host not in allowed_hosts and ref_host != req_host:\n return jsonify({'ok': False, 'error': 'Unauthorized request origin.'}), 403\n\n payload = request.get_json(force=True, silent=True) or {}\n report_id = f\"AIR-{time.strftime('%Y%m%d-%H%M%S')}-{uuid.uuid4().hex[:6].upper()}\"\n\n def _clean_str(key: str, limit: int = 4000) -> str:\n return str(payload.get(key, '')).strip()[:limit]\n\n model = _clean_str('model', 80)\n category = _clean_str('category', 120)\n snippet = _clean_str('snippet', 2000)\n details = _clean_str('details', 4000)\n contact = _clean_str('contact', 200)\n files = payload.get('files') or []\n if not isinstance(files, list):\n files = []\n files = [str(item).strip()[:260] for item in files[:20] if str(item).strip()]\n\n if not details:\n return jsonify({'ok': False, 'error': 'Please describe the issue.'}), 400\n\n LOGS_DIR.mkdir(exist_ok=True)\n record = {\n 'report_id': report_id,\n 'created_at': time.strftime('%Y-%m-%d %H:%M:%S'),\n 'category': category or 'inappropriate-output',\n 'model': model,\n 'files': files,\n 'snippet': snippet,\n 'details': details,\n 'contact': contact,\n }\n with open(AI_REPORTS_FILE, 'a', encoding='utf-8') as handle:\n handle.write(json.dumps(record, ensure_ascii=True) + '\\n')\n\n issue_title = url_quote(f'AI content report: {report_id}')\n issue_body = url_quote(\n f\"Report ID: {report_id}\\n\"\n f\"Category: {record['category']}\\n\"\n f\"Model: {model or 'not provided'}\\n\"\n f\"Snippet:\\n{snippet or 'not provided'}\\n\\n\"\n f\"Details:\\n{details}\\n\\n\"\n f\"Contact: {contact or 'not provided'}\\n\"\n )\n issue_url = f'{SUPPORT_ISSUES_URL}?title={issue_title}&body={issue_body}'\n report_text = (\n f\"Report ID: {report_id}\\n\"\n f\"Category: {record['category']}\\n\"\n f\"Model: {model or 'not provided'}\\n\"\n f\"Snippet:\\n{snippet or 'not provided'}\\n\\n\"\n f\"Details:\\n{details}\\n\\n\"\n f\"Contact: {contact or 'not provided'}\"\n )\n email_subject = url_quote(f'CVToposheet AI content report: {report_id}')\n email_body = url_quote(report_text)\n email_url = f'mailto:{SUPPORT_EMAIL}?subject={email_subject}&body={email_body}'\n return jsonify({\n 'ok': True,\n 'report_id': report_id,\n 'issue_url': issue_url,\n 'report_text': report_text,\n 'support_email': SUPPORT_EMAIL,\n 'email_url': email_url,\n })\n except Exception as exc:\n return jsonify({'ok': False, 'error': f'Failed to save report: {exc}'}), 500\n\n\n@app.route('/upload', methods=['POST'])\ndef upload():\n files = request.files.getlist('map_file')\n model = request.form.get('model', 'best')\n\n if not files or not files[0].filename:\n return redirect('/')\n\n session_id = uuid.uuid4().hex[:12]\n session_maps = _USER_DATA_DIR / 'maps' / session_id\n session_maps.mkdir(parents=True, exist_ok=True)\n\n saved_names = []\n for f in files:\n if f.filename and Path(f.filename).suffix.lower() in ALLOWED_EXT:\n safe_name = secure_filename(f.filename)\n dest = session_maps / safe_name\n f.save(str(dest))\n saved_names.append(safe_name)\n\n if not saved_names:\n return redirect('/')\n\n combined = ','.join(saved_names)\n return redirect(f'/process/{url_quote(combined, safe=\"\")}?model={url_quote(model, safe=\"\")}&session={url_quote(session_id, safe=\"\")}')\n\n\n@app.route('/kill', methods=['POST'])\ndef kill_process():\n global _current_proc\n killed = False\n if _current_proc is not None:\n try:\n if os.name == 'nt':\n # Force-kill process AND all its children (OCR workers, LLM calls etc.)\n subprocess.call(['taskkill', '/F', '/T', '/PID', str(_current_proc.pid)],\n stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n else:\n _current_proc.kill()\n killed = True\n except Exception:\n pass\n finally:\n _current_proc = None\n # Release the lock so the next map can be processed immediately\n try:\n _process_lock.release()\n except RuntimeError:\n pass # wasn't held\n return json.dumps({'killed': killed})\n\n\n@app.route('/restart', methods=['POST'])\ndef restart_server():\n \"\"\"Kill any running job and release the lock so the server is ready for a new upload.\"\"\"\n global _current_proc\n # Force-kill running subprocess and any children\n if _current_proc is not None:\n try:\n if os.name == 'nt':\n subprocess.call(['taskkill', '/F', '/T', '/PID', str(_current_proc.pid)],\n stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n else:\n _current_proc.kill()\n except Exception:\n pass\n finally:\n _current_proc = None\n # Release the processing lock so new jobs can run\n try:\n _process_lock.release()\n except RuntimeError:\n pass\n return json.dumps({'ok': True})\n\n\n@app.route('/stream/')\ndef stream(filename):\n model = request.args.get('model', 'best')\n session_id = request.args.get('session', '')\n return Response(\n stream_with_context(_stream_gen(filename, model, session_id)),\n mimetype='text/event-stream',\n headers={'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no'},\n )\n\n\ndef _stream_gen(filename, model, session_id):\n global _current_proc\n env_overrides = _MODEL_ENVS.get(model, _MODEL_ENVS['best'])\n\n # ── API key check ────────────────────────────────────────────────────────\n _KEY_NEEDED = {\n 'vertex': None, # uses service-account JSON, no API key\n 'gemini': 'GEMINI_API_KEY',\n 'groq': 'GROQ_API_KEY',\n 'openai': 'OPENAI_API_KEY',\n 'claude': 'CLAUDE_API_KEY',\n 'grok': 'GROK_API_KEY',\n 'openrouter': 'OPENROUTER_API_KEY',\n }\n provider = env_overrides.get('LLM_PROVIDER', 'vertex')\n required_key = _KEY_NEEDED.get(provider)\n if required_key and not os.environ.get(required_key, '').strip():\n friendly = {\n 'GEMINI_API_KEY': 'Gemini API Key',\n 'GROQ_API_KEY': 'Groq API Key',\n 'OPENAI_API_KEY': 'OpenAI API Key',\n 'CLAUDE_API_KEY': 'Claude API Key',\n 'GROK_API_KEY': 'Grok (xAI) API Key',\n 'OPENROUTER_API_KEY': 'OpenRouter API Key',\n }.get(required_key, required_key)\n msg = (f'❌ No {friendly} found. '\n f'Please go to ⚙️ Settings, enter your {friendly}, and save before processing.')\n yield f'data: {json.dumps({\"msg\": msg, \"error\": True})}\\n\\n'\n return\n # Also check service-account JSON for vertex LLM or GCV OCR (both need it)\n ocr_engine = env_overrides.get('OCR_ENGINE', os.environ.get('OCR_ENGINE', 'gcv')).lower()\n needs_gcp = (provider == 'vertex') or (ocr_engine == 'gcv')\n if needs_gcp:\n creds = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', '').strip()\n _gcp_found = (creds and Path(creds).exists()) or \\\n (_USER_DATA_DIR / 'service_account.json').exists() or \\\n (_USER_DATA_DIR / 'service_account2.json').exists() or \\\n (_USER_DATA_DIR / 'Service_account_Backup.json').exists() or \\\n (BASE_DIR / 'service_account.json').exists() or \\\n (BASE_DIR / 'Service_account_Backup.json').exists()\n if not _gcp_found:\n if provider == 'vertex':\n detail = 'Vertex AI LLM + Google Cloud Vision OCR both require'\n else:\n detail = 'Google Cloud Vision OCR requires'\n msg = (f'❌ No Google service-account JSON found. '\n f'{detail} a GCP service account. '\n f'Please go to ⚙️ Settings, upload your service_account.json, and save before processing.')\n yield f'data: {json.dumps({\"msg\": msg, \"error\": True})}\\n\\n'\n return\n # ── end API key check ────────────────────────────────────────────────────\n\n if not _process_lock.acquire(blocking=False):\n yield f'data: {json.dumps({\"msg\": \"Another map is already being processed. Please wait and try again.\"})}\\n\\n'\n yield f'data: {json.dumps({\"error\": True})}\\n\\n'\n return\n\n try:\n env = os.environ.copy()\n env.update(env_overrides)\n env['PYTHONIOENCODING'] = 'utf-8'\n env['PYTHONUNBUFFERED'] = '1'\n\n # Local OCR (EasyOCR/Tesseract) can trigger native DLL crashes when\n # BLAS/OpenMP thread pools over-subscribe in frozen builds.\n if env_overrides.get('OCR_ENGINE', '').lower() != 'gcv':\n # Force safe single-thread native runtime settings for local OCR.\n # setdefault is not enough because user/system env may already set\n # aggressive OpenMP/BLAS values that crash frozen EasyOCR/Torch.\n env['OMP_NUM_THREADS'] = '1'\n env['OPENBLAS_NUM_THREADS'] = '1'\n env['MKL_NUM_THREADS'] = '1'\n env['NUMEXPR_NUM_THREADS'] = '1'\n env['TORCH_NUM_THREADS'] = '1'\n env['TORCH_NUM_INTEROP_THREADS'] = '1'\n env['OMP_WAIT_POLICY'] = 'PASSIVE'\n env['KMP_AFFINITY'] = 'disabled'\n env['KMP_BLOCKTIME'] = '0'\n env['KMP_DUPLICATE_LIB_OK'] = 'TRUE'\n env['OPENCV_OPENCL_RUNTIME'] = 'disabled'\n env['OPENBLAS_CORETYPE'] = 'GENERIC'\n env['MKL_THREADING_LAYER'] = 'SEQUENTIAL'\n env['MKL_ENABLE_INSTRUCTIONS'] = 'COMPATIBLE'\n env['OMP_DYNAMIC'] = 'FALSE'\n env['PYTORCH_JIT'] = '0'\n env['GOMP_SPINCOUNT'] = '0'\n env['GOMP_STACKSIZE'] = '65536'\n\n session_maps_dir = str(_USER_DATA_DIR / 'maps' / session_id)\n session_results_dir = str(_USER_DATA_DIR / 'results' / session_id)\n session_logs_dir = str(_USER_DATA_DIR / 'logs' / session_id)\n Path(session_maps_dir).mkdir(parents=True, exist_ok=True)\n Path(session_results_dir).mkdir(parents=True, exist_ok=True)\n Path(session_logs_dir).mkdir(parents=True, exist_ok=True)\n env['MAPS_FOLDER'] = session_maps_dir\n env['RESULTS_FOLDER'] = session_results_dir\n env['LOGS_FOLDER'] = session_logs_dir\n results_url = f'/results/{session_id}'\n\n yield f'data: {json.dumps({\"msg\": f\"Starting processing for {filename}...\"})}\\n\\n'\n\n for label, script in [('pipeline', 'process_new_maps.py'),\n ('export', 'export_table.py')]:\n if label == 'export':\n yield f'data: {json.dumps({\"msg\": \"Exporting results table...\"})}\\n\\n'\n\n proc = subprocess.Popen(\n [sys.executable, script],\n cwd=str(BASE_DIR),\n stdout=subprocess.PIPE,\n stderr=subprocess.STDOUT,\n text=True,\n encoding='utf-8',\n errors='replace',\n bufsize=1,\n env=env,\n )\n _current_proc = proc\n\n # Read subprocess output in a background thread so we can send\n # SSE keepalive heartbeats when the process is silent (e.g. GCV\n # API calls), preventing the browser from closing the connection.\n line_q = queue.Queue()\n def _reader(pipe, q):\n for ln in iter(pipe.readline, ''):\n q.put(ln)\n q.put(None) # sentinel\n t = threading.Thread(target=_reader, args=(proc.stdout, line_q), daemon=True)\n t.start()\n\n while True:\n try:\n line = line_q.get(timeout=15)\n except queue.Empty:\n # No output for 15 s — send a keepalive comment so the\n # browser knows the connection is still alive.\n yield ': keepalive\\n\\n'\n continue\n if line is None: # sentinel — subprocess closed stdout\n break\n line = line.rstrip()\n if line:\n yield f'data: {json.dumps({\"msg\": line})}\\n\\n'\n\n proc.wait()\n _current_proc = None\n if label == 'pipeline' and proc.returncode not in (0, None):\n yield f'data: {json.dumps({\"msg\": f\"[ERROR] Pipeline exited with code {proc.returncode} — check logs\", \"error\": True})}\\n\\n'\n return\n\n # Merge this session into the global DB so Map Database stays up to date\n yield 'data: {\"msg\": \"Saving to Map Database...\"}\\n\\n'\n _merge_session_to_global(session_id)\n\n session_rows = _session_feature_count(session_id)\n if session_rows == 0:\n warn_msg = (\n 'Processing completed, but 0 features were extracted. '\n 'View Results may look empty. Try a clearer map, higher contrast scan, '\n 'or run with GCV OCR for stronger detection.'\n )\n yield f'data: {json.dumps({\"done\": True, \"redirect\": results_url, \"empty\": True, \"msg\": warn_msg})}\\n\\n'\n else:\n ok_msg = f'Done! Extracted {session_rows} feature(s). Click View Results.'\n yield f'data: {json.dumps({\"done\": True, \"redirect\": results_url, \"msg\": ok_msg})}\\n\\n'\n\n finally:\n try:\n _process_lock.release()\n except RuntimeError:\n pass\n\n\n@app.route('/process/')\ndef process(filename):\n model = request.args.get('model', 'best')\n session_id = request.args.get('session', '')\n return _process_html(filename, model, session_id)\n\n\n@app.route('/get_env')\ndef get_env():\n # Only allow requests originating from the app's own pages (pywebview or localhost).\n # Direct browser navigation (typing the URL) sends no Referer — block it.\n ref = request.referrer or ''\n if not (ref.startswith('http://127.0.0.1:') or ref.startswith('http://localhost:')):\n return '', 403\n keys = ['GROQ_API_KEY', 'GEMINI_API_KEY', 'GEMINI_API_KEY_2', 'OPENAI_API_KEY',\n 'CLAUDE_API_KEY', 'GROK_API_KEY', 'GOOGLE_API_KEY', 'OPENROUTER_API_KEY',\n 'OPENROUTER_MODEL', 'GOOGLE_APPLICATION_CREDENTIALS',\n 'VERTEX_PROJECT', 'VERTEX_LOCATION', 'VERTEX_MODEL']\n env_data = {k: os.environ.get(k, '') for k in keys}\n\n # Migrate legacy OpenRouter default model to the new recommended auto router.\n old_openrouter_default = 'meta-llama/llama-3.3-70b-instruct:free'\n or_model = env_data.get('OPENROUTER_MODEL', '').strip().lower()\n if (not or_model) or (or_model == old_openrouter_default):\n env_data['OPENROUTER_MODEL'] = 'openrouter/auto'\n os.environ['OPENROUTER_MODEL'] = 'openrouter/auto'\n try:\n _write_env({'OPENROUTER_MODEL': 'openrouter/auto'})\n except Exception:\n pass\n\n return json.dumps(env_data)\n\n\n@app.route('/results', methods=['GET', 'HEAD'])\n@app.route('/results/', methods=['GET', 'HEAD'])\ndef results(session_id=None):\n if session_id:\n out = _USER_DATA_DIR / 'results' / session_id / 'table_export.html'\n res_dir = str(_USER_DATA_DIR / 'results' / session_id)\n else:\n out = RESULTS_DIR / 'table_export.html'\n res_dir = str(RESULTS_DIR)\n\n # HEAD request (used by the polling check in the processing page)\n # Only return 200 if the HTML already exists — don't generate on HEAD\n if request.method == 'HEAD':\n if out.exists():\n return '', 200\n return '', 404\n\n # If HTML not yet generated, try to generate it on-demand from the DB\n # (also runs when DB doesn't exist yet — export_table handles empty/missing DB gracefully)\n if not out.exists():\n try:\n Path(res_dir).mkdir(parents=True, exist_ok=True)\n env = os.environ.copy()\n env['RESULTS_FOLDER'] = res_dir\n env['PYTHONIOENCODING'] = 'utf-8'\n env['PYTHONUNBUFFERED'] = '1'\n subprocess.run(\n [sys.executable, 'export_table.py'],\n cwd=str(BASE_DIR), env=env,\n timeout=60, capture_output=True\n )\n except Exception:\n pass\n\n if out.exists():\n return send_file(str(out))\n return ('

'\n 'No results yet – please upload and process a map first.'\n ' ← Go back

'), 404\n\n\n@app.route('/delete_maps', methods=['POST', 'OPTIONS'])\ndef delete_maps():\n # Allow CORS so the page can call this even when opened as file://\n if request.method == 'OPTIONS':\n resp = app.response_class(status=204)\n resp.headers['Access-Control-Allow-Origin'] = '*'\n resp.headers['Access-Control-Allow-Headers'] = 'Content-Type'\n resp.headers['Access-Control-Allow-Methods'] = 'POST'\n return resp\n\n data = request.get_json(silent=True) or {}\n maps_to_delete = data.get('maps', [])\n if not maps_to_delete:\n resp = app.response_class(json.dumps({'ok': False, 'error': 'No maps specified'}),\n mimetype='application/json')\n resp.headers['Access-Control-Allow-Origin'] = '*'\n return resp, 400\n\n deleted = 0\n db_path = RESULTS_DIR / 'toposheet.db'\n if db_path.exists():\n try:\n with sqlite3.connect(str(db_path)) as con:\n placeholders = ','.join('?' * len(maps_to_delete))\n cur = con.execute(\n f'DELETE FROM features WHERE map_name IN ({placeholders})',\n maps_to_delete\n )\n deleted = cur.rowcount\n con.commit()\n except Exception as e:\n resp = app.response_class(json.dumps({'ok': False, 'error': str(e)}),\n mimetype='application/json')\n resp.headers['Access-Control-Allow-Origin'] = '*'\n return resp, 500\n\n # Regenerate the HTML export so it reflects the deletion\n try:\n env = os.environ.copy()\n env['RESULTS_FOLDER'] = str(RESULTS_DIR)\n env['PYTHONIOENCODING'] = 'utf-8'\n result = subprocess.run([sys.executable, 'export_table.py'],\n cwd=str(BASE_DIR), env=env, timeout=60, capture_output=True)\n if result.returncode != 0:\n print('[export] export_table.py failed:', result.stderr.decode(errors='replace'))\n except Exception as e:\n print('[export] export_table.py exception:', e)\n\n resp = app.response_class(\n json.dumps({'ok': True, 'deleted_rows': deleted, 'maps': len(maps_to_delete)}),\n mimetype='application/json'\n )\n resp.headers['Access-Control-Allow-Origin'] = '*'\n return resp\n\n\n# ── Welcome / first-run onboarding page ──────────────────────────────────────\n_WELCOME_TEMPLATE = \"\"\"\n\n\n\n\nWelcome to CV-Toposheet\n\n\n\n
\n\n \n
\n
\n
CV-Toposheet
\n
AI-Powered Historical Toposheet Digitization
\n
\n
Step 1 of 4
\n
\n
\n\n \n
\n\n \n
\n
\n
📿
\n
Welcome to CV-Toposheet!
\n
Automatically extract & digitize geographic feature names from scanned historical topographic maps.
\n
\n
1
Upload
Drop a scanned toposheet (JPG / PNG / TIF)
\n
2
OCR
AI reads all text from every tile of the map
\n
3
Clean
LLM corrects errors & classifies features
\n
4
Search
Query & export the full results database
\n
\n
🕐 This setup screen appears once only. Takes about 60 seconds. You can revisit instructions anytime in Settings → Help.
\n
\n
\n\n \n
\n
\n
🔑
\n
Step 1: Add an LLM API Key
\n
Required for feature extraction. Go to ⚙ Settings on the home screen and paste at least one key.
\n
\n
\n
⚡
\n
\n
Groq API Key FREE & FASTEST
\n
No credit card needed. Best for first-time users.
\n
🔗 console.groq.com → Sign up → API Keys → Create
\n
\n
\n
\n
or
\n
\n
\n
💫
\n
\n
Gemini API Key FREE
\n
Google Gemini Flash. Free tier is generous.
\n
🔗 aistudio.google.com/apikey → Create API key
\n
\n
\n
\n
or
\n
\n
\n
💰
\n
\n
Other API Keys PAID
\n
Claude (Anthropic), Grok (xAI), OpenAI, and more. Add via ⚙ Settings after setup.
\n
🔗 anthropic.com  |  x.ai/api  |  platform.openai.com
\n
\n
\n
\n
⚠️ Without an API key the pipeline runs but produces wrong results: blank Feature Types, 0.50 confidence, noise rows.
\n
\n
\n\n \n
\n
\n
☁️
\n
Step 2: Google Cloud Vision JSON
\n
Strongly recommended for best accuracy. Enables premium OCR via Google Cloud Vision API. Without it, the app uses EasyOCR (offline, slower, less accurate).
\n
\n
How to get the JSON file
\n
\n 1. Go to console.cloud.google.com
\n 2. Enable Cloud Vision API (APIs & Services → Library)
\n 3. IAM & Admin → Service Accounts → Create → Keys → Add Key → JSON
\n 4. File downloads automatically\n
\n
\n
\n
\n
📄
\n
\n
Upload in Settings
\n
Click ⚙ Settings → Google Cloud → Upload GCP Service Account JSON and select the downloaded file.
\n
\n
\n
\n
✅ For best accuracy, set this up now. You can also add it later via Settings → Help → Google Cloud.
\n
⭐ For even better results, also enable Vertex AI API in Google Cloud Console (APIs & Services → Library → search “Vertex AI” → Enable).
\n
\n
\n\n \n
\n
\n
🚀
\n
You’re Ready to Go!
\n
Here’s how to process your first map.
\n
\n
🖼️
Upload: Drag & drop a JPG / PNG / TIF toposheet onto the home screen (up to 500 MB)
\n
🎭
Choose Model: Select a processing model from the right panel (Best Quality recommended)
\n
▶️
Process: Click Process Map and watch live progress (5–30 min depending on map size)
\n
📊
Search & Export: Find results in Map Database. Export to CSV or Excel anytime.
\n
\n
\n
\n\n
\n\n \n
\n \n
\n
\n
\n
\n
\n
\n \n
\n\n
\n\n
\n\n\n\n\"\"\"\n\n\n# ── Landing page HTML ─────────────────────────────────────────────────────────\ndef _landing_html(feat_count: int, map_count: int) -> str:\n html = _LANDING_TEMPLATE\n html = html.replace('__FEAT_COUNT__', str(feat_count))\n html = html.replace('__MAP_COUNT__', str(map_count))\n return html\n\n\n_LANDING_TEMPLATE = \"\"\"\n\n\n\n\nCV-Toposheet - Map Digitization\n\n\n\n\n
\n
\n
CV-Toposheet
\n
Extracted Map Features
\n
\n
\n 📊 Map Database\n \n \n \n
\n
\n\n
\n
\n
\n
\n \n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n 🗺\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n
Upload Toposheet
\n
Drag & drop map images here, or click to browse
\n
JPG · PNG · TIF · Up to 500 MB each
\n
\n
\n
\n\n
\n
Choose Model
\n
\n \n \n \n \n \n \n \n \n \n
\n Best Quality: Uses Google Cloud Vision for OCR and Vertex AI Gemini for feature extraction. Requires internet & GCP credentials.\n
\n
\n \n \n
\n
\n
\n\n\n\n\n
\n
\n

🔒 Settings Locked

\n

Enter your 4-digit PIN to open Settings

\n
\n \n \n \n \n
\n
\n
\n \n
\n
\n
\n\n\n
\n
\n

🔒 Set PIN Lock

\n

Enter a 4-digit PIN

\n
\n \n \n \n \n
\n
\n
\n \n \n
\n
\n
\n\n \n
\n
\n
\n

⚠ Report AI Content

\n \n
\n
\n

Use this form to report inappropriate, unsafe, or clearly incorrect AI-generated content. The app saves a local report with a reference ID and gives you a direct GitHub issue link for the publisher.

\n
\n
\n \n \n
\n
\n \n \n
\n
\n \n \n
\n
\n \n \n
\n
\n \n \n
\n
\n \n \n
\n
\n
\n
\n \n \n \n \n
\n
\n
\n\n\n
\n
\n
\n

⚙ API Key Settings

\n
\n \n \n \n
\n
\n
\n \n\n
\n
LLM Providers
\n
\n \n
\n \n \n
\n
\n
\n \n
\n \n \n
\n
\n
\n \n
\n \n \n
\n
\n
\n \n
\n \n \n
\n
\n
\n \n
\n \n \n
\n
\n
\n \n
\n \n \n
\n
\n
\n \n
\n \n \n
\n
\n
\n\n
\n
Model Versions
\n
\n \n
\n \n
\n
\n
\n \n
\n \n
\n
\n
\n \n
\n \n
\n
\n
\n \n
\n \n
\n
\n
\n \n
\n \n
\n
\n
\n \n
\n \n
\n
\n
\n \n
\n \n
\n
\n
\n\n
\n
Google Cloud
\n
\n \n
\n \n \n
\n
\n
\n \n
\n \n
\n
\n
\n \n Saves as service_account.json - used for Google Cloud Vision OCR\n
\n \n \n
\n \n
\n
\n \n Saves as service_account.json - requires Vertex AI API enabled + Vertex AI User IAM role in GCP\n
\n \n \n
\n \n
\n
\n\n
\n
Google Vertex AI
\n
\n \n
\n \n
\n
\n
\n \n
\n \n
\n
\n
\n\n
\n
Custom / Extra API Keys
\n
\n
\n \n \n \n
\n
\n
\n
\n ✓ Saved to .env\n \n \n
\n
\n
\n\n\n
\n
\n
\n

ℹ  Help & Setup Guide

\n \n
\n
\n\n \n
📿 What is CV-Toposheet?
\n

Automatically extract & digitize geographic feature names from scanned historical topographic maps.

\n
\n
1
Upload
Drop a scanned toposheet (JPG / PNG / TIF)
\n
2
OCR
AI reads all text from every tile of the map
\n
3
Clean
LLM corrects errors & classifies features
\n
4
Search
Query & export the full results database
\n
\n\n
\n\n \n
🔑 Step 1: Add an LLM API Key
\n

Required for feature extraction. Paste at least one key in ⚙ Settings.

\n
\n
⚡ Groq API Key FREE & FASTEST
\n
No credit card needed. Best for first-time users.
\n
🔗 console.groq.com → Sign up → API Keys → Create
\n
\n
or
\n
\n
💫 Gemini API Key FREE
\n
Google Gemini Flash. Free tier is generous.
\n
🔗 aistudio.google.com/apikey → Create API key
\n
\n
or
\n
\n
💰 Other API Keys PAID
\n
Claude (Anthropic), Grok (xAI), OpenAI, and more. Add via ⚙ Settings after setup.
\n
🔗 anthropic.com  |  x.ai/api  |  platform.openai.com
\n
\n
⚠️ Without an API key the pipeline runs but produces wrong results: blank Feature Types, 0.50 confidence, noise rows.
\n\n
\n\n \n
☁️ Step 2: Google Cloud Vision JSON
\n

Strongly recommended for best accuracy. Enables premium OCR via Google Cloud Vision API. Without it, the app uses EasyOCR (offline, slower, less accurate).

\n
\n
How to get the JSON file
\n
1. Go to console.cloud.google.com
2. Enable Cloud Vision API (APIs & Services → Library)
3. IAM & Admin → Service Accounts → Create → Keys → Add Key → JSON
4. File downloads automatically
\n
\n
\n
📄 Upload in Settings
\n
Click ⚙ Settings → Google Cloud → Upload GCP Service Account JSON and select the downloaded file.
\n
\n
⭐ For even better results, also enable Vertex AI API in Google Cloud Console (APIs & Services → Library → search “Vertex AI” → Enable).
\n\n
\n\n \n
⚙️ Model Versions (optional)
\n

Default model versions work for most users. Only change if you want a specific version.

\n
\n
Default models
\n
Groq: llama-3.1-8b-instant  |  Gemini: gemini-2.5-flash
OpenAI: gpt-4o-mini  |  Claude: claude-haiku-4-5
Grok: grok-3-mini  |  OpenRouter: openrouter/auto
\n
\n
💡 To use a different model, open ⚙ Settings → Model Versions, type the model name and save.
\n\n
\n\n \n
⚠️ Report AI-Generated Content
\n

If the AI generates inappropriate, unsafe, or clearly incorrect content, click ⚠ Report AI Content in the top bar. The app stores a report reference and opens a publisher issue link so the content can be reviewed.

\n\n
\n\n \n
🔒 PIN Lock (optional)
\n

Protect your API keys with a 4-digit PIN so no one else can open Settings.

\n
\n
Set a PIN
\n
Open ⚙ Settings → click the 🔓 lock icon (top right of Settings) → enter a 4-digit PIN → confirm.
\n
\n
\n
Remove a PIN
\n
Open ⚙ Settings → enter your current PIN → click Remove PIN.
\n
\n\n
\n\n \n
🚀 How to Process Your First Map
\n
🖼️
Upload: Drag & drop a JPG / PNG / TIF toposheet onto the home screen (up to 500 MB)
\n
🎭
Choose Model: Select a processing model from the right panel (Best Quality recommended)
\n
▶️
Process: Click Process Map and watch live progress (5–30 min depending on map size)
\n
📊
Search & Export: Find results in Map Database. Export to CSV or Excel anytime.
\n\n
\n
\n
\n\n\n\"\"\"\n\n\ndef _process_html(filename: str, model: str = 'best', session_id: str = '') -> str:\n import html as _html\n enc = url_quote(filename, safe='')\n model_enc = url_quote(model, safe='')\n session_enc = url_quote(session_id, safe='')\n safe_filename = _html.escape(filename)\n\n return f\"\"\"\n\n\n\n\nProcessing – {safe_filename}\n\n\n\n\n
\n
\n
CV-Toposheet
\n
Extracted Map Features
\n
\n \n
\n\n
\n
\n
\n
\n
\n Processing {safe_filename}\n
\n
\n\n
\n
1 · Tiling
\n
2 · OCR
\n
3 · Grid
\n
4 · LLM
\n
5 · Database
\n
Export
\n
\n\n
\n\n
\n 📊  View Results\n ←  Upload Another Map\n
\n
\n
\n\n\n\n\"\"\"\n\n\n# ── Entry point ───────────────────────────────────────────────────────────────\nif __name__ == '__main__':\n import logging\n # Kill any leftover processes on port 5000 before binding\n if os.name == 'nt':\n try:\n result = subprocess.check_output(\n 'netstat -ano | findstr \"127.0.0.1:5000\" | findstr LISTENING',\n shell=True, stderr=subprocess.DEVNULL\n ).decode()\n for line in result.strip().splitlines():\n parts = line.split()\n pid = parts[-1] if parts else None\n if pid and pid.isdigit() and int(pid) != os.getpid():\n subprocess.call(['taskkill', '/F', '/PID', pid],\n shell=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n except Exception:\n pass\n log = logging.getLogger('werkzeug')\n log.setLevel(logging.ERROR)\n print()\n print('=' * 52)\n print(' CV-Toposheet Map Digitization Interface')\n print(' Open your browser: http://localhost:5000')\n print('=' * 52)\n print()\n app.run(debug=False, host='127.0.0.1', port=5000, threaded=True)\n"}}}}