1 line
30 KiB
JSON
1 line
30 KiB
JSON
{"repo": "christophertetteh04/ghanadatahub", "files": {"scripts/scrape_ghana_gov_expanded.py": {"126d73a241dde9b45852bbbacd6a33faf5797af5": {"t": 1785768920.5203593, "n": 28242, "h": "97aff12cf69a6147", "c": "#!/usr/bin/env python3\n\"\"\"Expanded GhanaDataHub source monitor and uploader.\n\nUsage:\n python3 scripts/scrape_ghana_gov_expanded.py\n python3 scripts/scrape_ghana_gov_expanded.py --sources godi_ckan bog_statistical\n python3 scripts/scrape_ghana_gov_expanded.py --dry-run\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport io\nimport os\nimport tempfile\nimport time\nimport warnings\nimport zipfile\nfrom html.parser import HTMLParser\nfrom pathlib import Path\nfrom typing import Callable\nfrom urllib.parse import urljoin, urlparse\n\nimport requests\nfrom urllib3.exceptions import InsecureRequestWarning\n\n\nUSER_AGENT = \"GhanaDataHub/1.0 (data research contact@ghanadatahub.com)\"\nDEFAULT_API_BASE = os.getenv(\"GHANADATAHUB_API_BASE\") or os.getenv(\"GDH_API_BASE\") or \"http://localhost:8000\"\nMAX_PACKAGES_PER_RUN = 50\nwarnings.filterwarnings(\"ignore\", category=InsecureRequestWarning)\n\n\nclass AnchorParser(HTMLParser):\n def __init__(self) -> None:\n super().__init__()\n self.anchors: list[dict] = []\n self._current: dict | None = None\n\n def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:\n if tag.lower() == \"a\":\n attr_map = dict(attrs)\n href = attr_map.get(\"href\")\n if href:\n self._current = {\"href\": href, \"text\": \"\"}\n\n def handle_data(self, data: str) -> None:\n if self._current is not None:\n self._current[\"text\"] += data\n\n def handle_endtag(self, tag: str) -> None:\n if tag.lower() == \"a\" and self._current is not None:\n self._current[\"text\"] = self._current[\"text\"].strip()\n self.anchors.append(self._current)\n self._current = None\n\n\ndef extract_links(html: str) -> list[dict]:\n try:\n from bs4 import BeautifulSoup\n\n soup = BeautifulSoup(html, \"html.parser\")\n return [{\"href\": a[\"href\"], \"text\": a.get_text(strip=True)} for a in soup.find_all(\"a\", href=True)]\n except ImportError:\n parser = AnchorParser()\n parser.feed(html)\n return parser.anchors\n\n\ndef normalise_api_base(api_base: str) -> str:\n return api_base.rstrip(\"/\")\n\n\ndef load_local_env() -> None:\n \"\"\"Load simple KEY=value pairs from scripts/.env and backend/.env if present.\"\"\"\n for env_path in (Path(__file__).resolve().parent / \".env\", Path(__file__).resolve().parent.parent / \"backend\" / \".env\"):\n if not env_path.exists():\n continue\n for line in env_path.read_text().splitlines():\n line = line.strip()\n if not line or line.startswith(\"#\") or \"=\" not in line:\n continue\n if \"GDH_EMAIL=\" in line and not line.startswith(\"GDH_EMAIL=\"):\n os.environ.setdefault(\"GDH_EMAIL\", line.split(\"GDH_EMAIL=\", 1)[1].strip())\n continue\n key, value = line.split(\"=\", 1)\n os.environ.setdefault(key.strip(), value.strip())\n\n\nload_local_env()\n\n\ndef absolute_url(base_url: str, href: str) -> str:\n if href.startswith(\"//\"):\n return \"https:\" + href\n if href.startswith(\"http://\") or href.startswith(\"https://\"):\n return href\n return urljoin(base_url, href)\n\n\ndef safe_filename(title: str, fallback_ext: str = \".csv\") -> str:\n stem = \"\".join(ch if ch.isalnum() else \"_\" for ch in title.lower()).strip(\"_\")\n stem = \"_\".join(part for part in stem.split(\"_\") if part)[:80] or \"dataset\"\n if \".\" not in Path(stem).name:\n return f\"{stem}{fallback_ext}\"\n return stem\n\n\ndef content_type_for_url(url: str) -> str:\n path = urlparse(url).path.lower()\n if path.endswith(\".csv\"):\n return \"text/csv\"\n if path.endswith(\".xlsx\"):\n return \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\"\n if path.endswith(\".xls\"):\n return \"application/vnd.ms-excel\"\n if path.endswith(\".json\"):\n return \"application/json\"\n if path.endswith(\".pdf\"):\n return \"application/pdf\"\n return \"application/octet-stream\"\n\n\ndef scrape_ckan_api(source_config: dict, session: requests.Session, existing_titles: set[str]) -> list[dict]:\n \"\"\"Scrape any CKAN-powered open data portal (GODI, HDX).\"\"\"\n api_url = source_config[\"api_url\"]\n resp = session.get(f\"{api_url}package_list\", timeout=30, verify=False)\n resp.raise_for_status()\n packages = resp.json()[\"result\"]\n new_datasets = []\n\n for pkg_name in packages[:MAX_PACKAGES_PER_RUN]:\n pkg_resp = session.get(f\"{api_url}package_show?id={pkg_name}\", timeout=15, verify=False)\n if not pkg_resp.ok:\n continue\n pkg = pkg_resp.json().get(\"result\", {})\n if not isinstance(pkg, dict):\n continue\n\n if \"country_filter\" in source_config:\n country = source_config[\"country_filter\"]\n pkg_groups = [\n g.get(\"name\", \"\") if isinstance(g, dict) else str(g)\n for g in pkg.get(\"groups\", [])\n ]\n if country.lower() not in \" \".join(pkg_groups).lower():\n continue\n\n for resource in pkg.get(\"resources\", []):\n if not isinstance(resource, dict):\n continue\n fmt = (resource.get(\"format\") or \"\").upper()\n if fmt in (\"CSV\", \"XLSX\", \"XLS\", \"JSON\"):\n title = f\"{source_config['name']}: {pkg.get('title') or pkg_name}\"\n if title.lower().strip() in existing_titles:\n continue\n new_datasets.append(\n {\n \"title\": title,\n \"url\": resource[\"url\"],\n \"description\": pkg.get(\"notes\", \"\"),\n \"attribution\": source_config[\"attribution\"],\n \"source_url\": source_config[\"url\"],\n \"licence\": source_config[\"licence\"],\n \"category\": source_config[\"category\"],\n }\n )\n break\n return new_datasets\n\n\ndef scrape_pdf_link_monitor(source_config: dict, session: requests.Session, existing_titles: set[str]) -> list[dict]:\n \"\"\"Monitor a publications page for new PDF or Excel links.\"\"\"\n resp = session.get(source_config[\"publications_url\"], timeout=30, verify=False)\n if not resp.ok:\n return []\n\n links = extract_links(resp.text)\n keywords = source_config.get(\"keywords\", [])\n new_datasets = []\n\n for link in links:\n href = link[\"href\"]\n link_text = link.get(\"text\", \"\").lower()\n ext = href.lower().split(\"?\")[0].split(\".\")[-1]\n if ext not in (\"pdf\", \"xlsx\", \"xls\", \"csv\"):\n continue\n if keywords and not any(kw in link_text or kw in href.lower() for kw in keywords):\n continue\n\n href = absolute_url(source_config[\"publications_url\"], href)\n title = f\"{source_config['name']}: {link.get('text') or href.split('/')[-1]}\"[:200]\n if title.lower().strip() in existing_titles:\n continue\n\n new_datasets.append(\n {\n \"title\": title,\n \"url\": href,\n \"description\": f\"Published by {source_config['name']}. Original source: {href}\",\n \"attribution\": source_config[\"attribution\"],\n \"source_url\": source_config[\"url\"],\n \"licence\": source_config[\"licence\"],\n \"category\": source_config[\"category\"],\n }\n )\n return new_datasets\n\n\ndef scrape_bog_excel(source_config: dict, session: requests.Session, existing_titles: set[str]) -> list[dict]:\n \"\"\"Monitor BOG publications page for new Excel files.\"\"\"\n resp = session.get(source_config[\"publications_url\"], timeout=30, verify=False)\n if not resp.ok:\n return []\n\n links = extract_links(resp.text)\n new_datasets = []\n\n for link in links:\n href = link[\"href\"]\n if not href.lower().split(\"?\")[0].endswith((\".xlsx\", \".xls\")):\n continue\n href = absolute_url(source_config[\"publications_url\"], href)\n label = link.get(\"text\") or href.split(\"/\")[-1]\n title = f\"Bank of Ghana Statistical Data: {label}\"[:200]\n if title.lower().strip() in existing_titles:\n continue\n new_datasets.append(\n {\n \"title\": title,\n \"url\": href,\n \"description\": \"Bank of Ghana macroeconomic and financial data.\",\n \"attribution\": source_config[\"attribution\"],\n \"source_url\": source_config[\"url\"],\n \"licence\": source_config[\"licence\"],\n \"category\": source_config[\"category\"],\n }\n )\n return new_datasets\n\n\ndef scrape_worldbank_api(source_config: dict, session: requests.Session, existing_titles: set[str]) -> list[dict]:\n \"\"\"Create one downloadable World Bank API CSV URL per configured Ghana indicator.\"\"\"\n new_datasets = []\n for indicator in source_config.get(\"indicators\", []):\n title = f\"{source_config['name']}: {indicator}\"\n if title.lower().strip() in existing_titles:\n continue\n url = (\n f\"{source_config['api_url']}country/{source_config['country_code']}/indicator/\"\n f\"{indicator}?downloadformat=csv\"\n )\n new_datasets.append(\n {\n \"title\": title,\n \"url\": url,\n \"description\": f\"World Bank indicator {indicator} for Ghana.\",\n \"attribution\": source_config[\"attribution\"],\n \"source_url\": source_config[\"url\"],\n \"licence\": source_config[\"licence\"],\n \"category\": source_config[\"category\"],\n }\n )\n return new_datasets\n\n\ndef scrape_not_implemented(source_config: dict, session: requests.Session, existing_titles: set[str]) -> list[dict]:\n \"\"\"Placeholder for specialized APIs that need endpoint-specific query design.\"\"\"\n print(f\" Method {source_config['scrape_method']} registered; endpoint-specific scraper pending\")\n return []\n\n\nSOURCE_REGISTRY: dict[str, dict] = {\n # TIER 1: OPEN DATA APIS\n \"godi_ckan\": {\n \"name\": \"Ghana Open Data Initiative\",\n \"url\": \"https://data.gov.gh\",\n \"api_url\": \"https://data.gov.gh/api/3/action/\",\n \"licence\": \"Open Data Commons Attribution Licence (ODC-By 1.0)\",\n \"licence_url\": \"https://opendatacommons.org/licenses/by/1.0/\",\n \"attribution\": \"Ghana Open Data Initiative (GODI), Government of Ghana. data.gov.gh\",\n \"category\": \"Multiple\",\n \"update_frequency\": \"irregular\",\n \"scrape_method\": \"ckan_api\",\n \"scrape_function\": scrape_ckan_api,\n },\n \"gss_statsbank\": {\n \"name\": \"Ghana Statistical Service - StatsBank\",\n \"url\": \"https://statsbank.statsghana.gov.gh\",\n \"publications_url\": \"https://statsghana.gov.gh/statistical-publications\",\n \"licence\": \"Open Government Licence - Ghana\",\n \"attribution\": \"Ghana Statistical Service (GSS). statsghana.gov.gh\",\n \"category\": \"Demographics\",\n \"update_frequency\": \"monthly\",\n \"scrape_method\": \"publications_monitor\",\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"bog_statistical\": {\n \"name\": \"Bank of Ghana - Statistical Bulletin\",\n \"url\": \"https://www.bog.gov.gh\",\n \"publications_url\": \"https://www.bog.gov.gh/monetary-policy/summary-of-economic-and-financial-data/\",\n \"licence\": \"Open for public use under BOG Act 2002 Section 55\",\n \"attribution\": \"Bank of Ghana (BOG). bog.gov.gh. Statistical Bulletin.\",\n \"category\": \"Economy and Finance\",\n \"update_frequency\": \"monthly\",\n \"scrape_method\": \"excel_link_monitor\",\n \"scrape_function\": scrape_bog_excel,\n },\n \"hdx_ghana_health\": {\n \"name\": \"Humanitarian Data Exchange - Ghana Health\",\n \"url\": \"https://data.humdata.org\",\n \"api_url\": \"https://data.humdata.org/api/3/action/\",\n \"licence\": \"Creative Commons Attribution for Intergovernmental Organisations\",\n \"attribution\": \"WHO / Ghana Health Service via HDX. data.humdata.org\",\n \"category\": \"Health\",\n \"update_frequency\": \"varies\",\n \"scrape_method\": \"ckan_api\",\n \"org_filter\": \"who\",\n \"country_filter\": \"GHA\",\n \"scrape_function\": scrape_ckan_api,\n },\n # TIER 2: PDF/EXCEL PUBLICATION MONITORS\n \"ec_voter\": {\n \"name\": \"Electoral Commission of Ghana\",\n \"url\": \"https://www.ec.gov.gh\",\n \"publications_url\": \"https://www.ec.gov.gh/publications.html\",\n \"licence\": \"Public domain - EC publications are public records\",\n \"attribution\": \"Electoral Commission of Ghana (EC). ec.gov.gh\",\n \"category\": \"Governance and Electoral\",\n \"update_frequency\": \"after_registration_periods\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"voter\", \"registration\", \"election\", \"results\", \"constituency\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"energy_commission\": {\n \"name\": \"Ghana Energy Commission\",\n \"url\": \"https://www.energycom.gov.gh\",\n \"publications_url\": \"https://www.energycom.gov.gh/energy-statistics\",\n \"licence\": \"Open for public use\",\n \"attribution\": \"Ghana Energy Commission. energycom.gov.gh. National Energy Statistics.\",\n \"category\": \"Energy and Infrastructure\",\n \"update_frequency\": \"annual\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"energy\", \"statistics\", \"balance\", \"electricity\", \"renewable\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"gra_revenue\": {\n \"name\": \"Ghana Revenue Authority\",\n \"url\": \"https://www.gra.gov.gh\",\n \"publications_url\": \"https://www.gra.gov.gh/resources/publications/\",\n \"licence\": \"Public domain\",\n \"attribution\": \"Ghana Revenue Authority (GRA). gra.gov.gh. Revenue Statistics.\",\n \"category\": \"Economy and Finance\",\n \"update_frequency\": \"quarterly\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"revenue\", \"tax\", \"domestic\", \"quarterly\", \"annual\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"mofep_budget\": {\n \"name\": \"Ministry of Finance - Budget Documents\",\n \"url\": \"https://www.mofep.gov.gh\",\n \"publications_url\": \"https://www.mofep.gov.gh/publications/\",\n \"licence\": \"Public domain - government publications\",\n \"attribution\": \"Ministry of Finance and Economic Planning (MoFEP). mofep.gov.gh.\",\n \"category\": \"Economy and Finance\",\n \"update_frequency\": \"annual_budget_quarterly_bulletin\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"budget\", \"fiscal\", \"policy\", \"midyear\", \"quarterly\", \"debt\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"cocobod\": {\n \"name\": \"Ghana Cocoa Board (COCOBOD)\",\n \"url\": \"https://www.cocobod.gh\",\n \"publications_url\": \"https://www.cocobod.gh/resources/publications\",\n \"licence\": \"Public domain\",\n \"attribution\": \"Ghana Cocoa Board (COCOBOD). cocobod.gh.\",\n \"category\": \"Agriculture and Food\",\n \"update_frequency\": \"seasonal\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"cocoa\", \"purchase\", \"production\", \"export\", \"price\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"ppa_procurement\": {\n \"name\": \"Public Procurement Authority\",\n \"url\": \"https://www.ppaghana.org\",\n \"publications_url\": \"https://www.ppaghana.org/index.php/resources/publications\",\n \"licence\": \"Public domain\",\n \"attribution\": \"Public Procurement Authority (PPA) Ghana. ppaghana.org.\",\n \"category\": \"Governance and Electoral\",\n \"update_frequency\": \"annual\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"procurement\", \"contract\", \"award\", \"annual\", \"statistics\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"ghanaports\": {\n \"name\": \"Ghana Ports and Harbours Authority\",\n \"url\": \"https://www.ghanaports.com.gh\",\n \"publications_url\": \"https://www.ghanaports.com.gh/about/statistics\",\n \"licence\": \"Public domain\",\n \"attribution\": \"Ghana Ports and Harbours Authority (GPHA). ghanaports.com.gh.\",\n \"category\": \"Trade and Commerce\",\n \"update_frequency\": \"monthly\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"port\", \"throughput\", \"trade\", \"container\", \"cargo\", \"vessels\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"nic_insurance\": {\n \"name\": \"National Insurance Commission\",\n \"url\": \"https://www.nicgh.org\",\n \"publications_url\": \"https://www.nicgh.org/publications\",\n \"licence\": \"Public domain\",\n \"attribution\": \"National Insurance Commission (NIC) Ghana. nicgh.org.\",\n \"category\": \"Economy and Finance\",\n \"update_frequency\": \"annual\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"insurance\", \"sector\", \"statistics\", \"premium\", \"market\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n \"ghana_health_service\": {\n \"name\": \"Ghana Health Service\",\n \"url\": \"https://ghs.gov.gh\",\n \"publications_url\": \"https://ghs.gov.gh/resources-hub\",\n \"licence\": \"Public domain - government health publications\",\n \"attribution\": \"Ghana Health Service (GHS). ghs.gov.gh.\",\n \"category\": \"Health\",\n \"update_frequency\": \"irregular\",\n \"scrape_method\": \"pdf_link_monitor\",\n \"keywords\": [\"health\", \"disease\", \"surveillance\", \"report\", \"guidelines\", \"statistics\"],\n \"scrape_function\": scrape_pdf_link_monitor,\n },\n # TIER 3: INTERNATIONAL SOURCES\n \"who_ghana\": {\n \"name\": \"World Health Organisation - Ghana\",\n \"url\": \"https://www.who.int/countries/gha\",\n \"data_url\": \"https://ghoapi.azureedge.net/api/\",\n \"licence\": \"CC BY-NC-SA 3.0 IGO\",\n \"attribution\": \"World Health Organisation (WHO). who.int/countries/gha.\",\n \"category\": \"Health\",\n \"update_frequency\": \"varies\",\n \"scrape_method\": \"who_api\",\n \"country_code\": \"GHA\",\n \"scrape_function\": scrape_not_implemented,\n },\n \"faostat_ghana\": {\n \"name\": \"FAOSTAT - Ghana Agricultural Data\",\n \"url\": \"https://www.fao.org/faostat\",\n \"api_url\": \"https://fenixservices.fao.org/faostat/api/v1/\",\n \"licence\": \"CC BY-NC-SA 3.0 IGO\",\n \"attribution\": \"Food and Agriculture Organisation (FAO). FAOSTAT. fao.org/faostat.\",\n \"category\": \"Agriculture and Food\",\n \"update_frequency\": \"annual\",\n \"scrape_method\": \"faostat_api\",\n \"country_code\": \"49\",\n \"scrape_function\": scrape_not_implemented,\n },\n \"comtrade_ghana\": {\n \"name\": \"UN Comtrade - Ghana Trade Data\",\n \"url\": \"https://comtrade.un.org\",\n \"api_url\": \"https://comtradeapi.un.org/data/v1/\",\n \"licence\": \"CC BY 4.0\",\n \"attribution\": \"UN Comtrade Database. comtrade.un.org.\",\n \"category\": \"Trade and Commerce\",\n \"update_frequency\": \"annual\",\n \"scrape_method\": \"comtrade_api\",\n \"country_code\": \"288\",\n \"scrape_function\": scrape_not_implemented,\n },\n \"world_bank_ghana\": {\n \"name\": \"World Bank Open Data - Ghana\",\n \"url\": \"https://data.worldbank.org/country/ghana\",\n \"api_url\": \"https://api.worldbank.org/v2/\",\n \"licence\": \"CC BY 4.0\",\n \"attribution\": \"World Bank Open Data. data.worldbank.org.\",\n \"category\": \"Multiple\",\n \"update_frequency\": \"annual\",\n \"scrape_method\": \"worldbank_api\",\n \"country_code\": \"GH\",\n \"indicators\": [\n \"NY.GDP.MKTP.CD\", \"NY.GDP.MKTP.KD.ZG\", \"FP.CPI.TOTL.ZG\",\n \"SP.POP.TOTL\", \"SL.UEM.TOTL.ZS\", \"SE.ADT.LITR.ZS\",\n \"SP.DYN.IMRT.IN\", \"SH.STA.MMRT\", \"SH.MED.BEDS.ZS\",\n \"EG.ELC.ACCS.ZS\", \"AG.PRD.CROP.XD\", \"TM.VAL.MRCH.CD.WT\",\n \"TX.VAL.MRCH.CD.WT\", \"GC.REV.TOTL.GD.ZS\", \"GC.DOD.TOTL.GD.ZS\",\n \"IT.NET.USER.ZS\", \"SH.XPD.CHEX.GD.ZS\", \"SE.XPD.TOTL.GD.ZS\",\n \"EN.ATM.CO2E.PC\", \"SP.URB.TOTL.IN.ZS\",\n ],\n \"scrape_function\": scrape_worldbank_api,\n },\n \"unicef_ghana\": {\n \"name\": \"UNICEF - Ghana Child Data\",\n \"url\": \"https://data.unicef.org\",\n \"api_url\": \"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/\",\n \"licence\": \"CC BY 4.0\",\n \"attribution\": \"UNICEF Data. data.unicef.org.\",\n \"category\": \"Health\",\n \"update_frequency\": \"annual\",\n \"scrape_method\": \"unicef_api\",\n \"country_code\": \"GHA\",\n \"scrape_function\": scrape_not_implemented,\n },\n}\n\n\ndef get_existing_titles(api_base: str, token: str | None) -> set[str]:\n if not token:\n return set()\n try:\n resp = requests.get(\n f\"{normalise_api_base(api_base)}/api/v1/datasets/?per_page=100\",\n headers={\"Authorization\": f\"Bearer {token}\"},\n timeout=30,\n )\n resp.raise_for_status()\n existing = resp.json()\n return {d[\"title\"].lower().strip() for d in existing.get(\"items\", [])}\n except Exception as exc:\n print(f\"WARNING: Could not fetch existing titles: {exc}\")\n return set()\n\n\ndef get_token(api_base: str, token: str | None = None, email: str | None = None, password: str | None = None) -> str | None:\n \"\"\"Use an explicit token, env token, or login credentials to get an upload token.\"\"\"\n token = token or os.getenv(\"GHANADATAHUB_TOKEN\") or os.getenv(\"GDH_TOKEN\")\n if token:\n return token\n\n email = email or os.getenv(\"GHANADATAHUB_EMAIL\") or os.getenv(\"GDH_EMAIL\")\n password = password or os.getenv(\"GHANADATAHUB_PASSWORD\") or os.getenv(\"GDH_PASSWORD\")\n if not email or not password:\n return None\n\n try:\n resp = requests.post(\n f\"{normalise_api_base(api_base)}/api/v1/auth/login\",\n json={\"email\": email, \"password\": password},\n timeout=30,\n )\n resp.raise_for_status()\n return resp.json().get(\"access_token\")\n except Exception as exc:\n print(f\"WARNING: Automatic login failed: {exc}\")\n return None\n\n\ndef upload_dataset(dataset: dict, api_base: str, token: str, session: requests.Session | None = None) -> bool:\n \"\"\"Download a remote file and upload it to GhanaDataHub.\"\"\"\n session = session or requests.Session()\n session.headers.setdefault(\"User-Agent\", USER_AGENT)\n remote_url = dataset[\"url\"]\n print(f\" Uploading {dataset['title']}\")\n\n try:\n remote_resp = session.get(remote_url, timeout=60, verify=False)\n remote_resp.raise_for_status()\n except Exception as exc:\n print(f\" FAILED download: {exc}\")\n return False\n\n content = remote_resp.content\n content_type = remote_resp.headers.get(\"Content-Type\") or content_type_for_url(remote_url)\n ext = Path(urlparse(remote_url).path).suffix or \".dat\"\n filename = safe_filename(dataset[\"title\"], ext)\n\n if \"zip\" in content_type.lower() or zipfile.is_zipfile(io.BytesIO(content)):\n with zipfile.ZipFile(io.BytesIO(content)) as zf:\n csv_name = next((name for name in zf.namelist() if name.lower().endswith(\".csv\")), None)\n if not csv_name:\n print(\" FAILED download: ZIP did not contain a CSV file\")\n return False\n content = zf.read(csv_name)\n content_type = \"text/csv\"\n filename = safe_filename(dataset[\"title\"], \".csv\")\n\n allowed_content_types = (\n \"text/csv\",\n \"application/csv\",\n \"application/json\",\n \"application/pdf\",\n \"application/vnd.ms-excel\",\n \"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\",\n )\n if not any(allowed in content_type.lower() for allowed in allowed_content_types):\n print(f\" SKIP unsupported download type: {content_type}\")\n return False\n\n with tempfile.NamedTemporaryFile(delete=False) as tmp:\n tmp.write(content)\n tmp_path = Path(tmp.name)\n\n try:\n with tmp_path.open(\"rb\") as fh:\n files = {\"file\": (filename, fh, content_type)}\n data = {\n \"title\": dataset[\"title\"],\n \"description\": (\n f\"{dataset.get('description', '')}\\n\\n\"\n f\"Attribution: {dataset['attribution']}\\n\"\n f\"Source: {dataset['source_url']}\"\n ).strip(),\n \"visibility\": \"public\",\n \"license\": dataset[\"licence\"],\n \"tags\": f\"ghana,government,{dataset['category'].lower().replace(' ', '-')}\",\n }\n resp = requests.post(\n f\"{normalise_api_base(api_base)}/api/v1/datasets/\",\n headers={\"Authorization\": f\"Bearer {token}\"},\n data=data,\n files=files,\n timeout=120,\n )\n if resp.status_code == 201:\n print(\" SUCCESS\")\n return True\n print(f\" FAILED upload: {resp.status_code} {resp.text[:300]}\")\n return False\n finally:\n tmp_path.unlink(missing_ok=True)\n\n\ndef run_expanded_scrapers(\n api_base: str = DEFAULT_API_BASE,\n token: str | None = None,\n sources: list[str] | None = None,\n dry_run: bool = False,\n email: str | None = None,\n password: str | None = None,\n) -> int:\n \"\"\"Run configured scrapers and upload new datasets.\"\"\"\n token = get_token(api_base, token=token, email=email, password=password)\n session = requests.Session()\n session.headers[\"User-Agent\"] = USER_AGENT\n existing_titles = get_existing_titles(api_base, token)\n sources_to_run = sources or list(SOURCE_REGISTRY.keys())\n total_new = 0\n\n for source_key in sources_to_run:\n config = SOURCE_REGISTRY.get(source_key)\n if not config:\n print(f\"Skipping unknown source: {source_key}\")\n continue\n\n print(f\"Checking {config['name']}...\")\n try:\n scrape_function: Callable = config[\"scrape_function\"]\n new = scrape_function(config, session, existing_titles)\n print(f\" Found {len(new)} new datasets\")\n if dry_run:\n for dataset in new[:10]:\n print(f\" DRY RUN: {dataset['title']} -> {dataset['url']}\")\n continue\n if not token:\n print(\" No GHANADATAHUB_TOKEN provided; skipping uploads\")\n continue\n for dataset in new:\n if upload_dataset(dataset, api_base, token, session=session):\n existing_titles.add(dataset[\"title\"].lower().strip())\n total_new += 1\n time.sleep(1)\n except Exception as exc:\n print(f\" ERROR scraping {source_key}: {exc}\")\n\n print(f\"\\nTotal new datasets uploaded: {total_new}\")\n return total_new\n\n\ndef parse_args() -> argparse.Namespace:\n parser = argparse.ArgumentParser(description=\"Run expanded GhanaDataHub scrapers.\")\n parser.add_argument(\"--api-base\", default=os.getenv(\"GHANADATAHUB_API_BASE\") or os.getenv(\"GDH_API_BASE\") or DEFAULT_API_BASE)\n parser.add_argument(\"--token\", default=os.getenv(\"GHANADATAHUB_TOKEN\") or os.getenv(\"GDH_TOKEN\"))\n parser.add_argument(\"--email\", default=os.getenv(\"GHANADATAHUB_EMAIL\") or os.getenv(\"GDH_EMAIL\"))\n parser.add_argument(\"--password\", default=os.getenv(\"GHANADATAHUB_PASSWORD\") or os.getenv(\"GDH_PASSWORD\"))\n parser.add_argument(\"--sources\", nargs=\"*\", help=\"Optional source keys to run\")\n parser.add_argument(\"--dry-run\", action=\"store_true\")\n parser.add_argument(\"--list-sources\", action=\"store_true\")\n return parser.parse_args()\n\n\ndef main() -> int:\n args = parse_args()\n if args.list_sources:\n for key, source in SOURCE_REGISTRY.items():\n print(f\"{key}: {source['name']} ({source['scrape_method']})\")\n return 0\n run_expanded_scrapers(\n api_base=args.api_base,\n token=args.token,\n sources=args.sources,\n dry_run=args.dry_run,\n email=args.email,\n password=args.password,\n )\n return 0\n\n\nif __name__ == \"__main__\":\n raise SystemExit(main())\n"}}}} |