1 line
46 KiB
JSON
1 line
46 KiB
JSON
{"repo": "a15280020511/evidence-data-center", "files": {"api-center/public-data-geospatial/public_data_geospatial_task.py": {"78aa917b54519ccdafe558954c2e45e7df773466": {"t": 1785768922.9300134, "n": 22436, "h": "cbb9cc0459e7b06b", "c": "#!/usr/bin/env python3\n\"\"\"Bounded, fixed-host public-data and geospatial provider.\"\"\"\nfrom __future__ import annotations\nimport json\nimport os\nimport sys\nimport time\nfrom pathlib import Path\nfrom typing import Any, Mapping\nfrom urllib.parse import urlparse\nimport requests\n\nHERE=Path(__file__).resolve().parent\nsys.path.insert(0,str(HERE.parent))\nfrom managed_provider_runtime import bounded_int, bytes_sha, finish_execution, load_json, provider_row, run_cli, utc_now, validate_ticket\nSCHEMA_PATH=HERE/'ticket.schema.json'; CATALOG_PATH=HERE/'provider-catalog.json'\n\nclass Spec:\n def __init__(self, method:str, url:str, params=None, json_body=None, data_body=None, headers=None, auth=None, credential_mode='none', response_kind='json'):\n self.method=method; self.url=url; self.params=params; self.json_body=json_body\n self.data_body=data_body; self.headers=headers; self.auth=auth\n self.credential_mode=credential_mode; self.response_kind=response_kind\n\ndef text(p:Mapping[str,Any],name:str,maximum:int=300,required:bool=False)->str:\n value=str(p.get(name) or '').strip()\n if required and not value: raise ValueError(f'{name} is required')\n if len(value)>maximum or any(ord(ch)<32 for ch in value): raise ValueError(f'{name} is invalid')\n return value\n\ndef secret(name:str,required:bool=True)->str:\n value=str(os.environ.get(name) or '').strip()\n if required and not value: raise RuntimeError(f'missing required backend secret or variable: {name}')\n return value\n\ndef integer(p:Mapping[str,Any],name:str,default:int,lo:int,hi:int)->int:\n return bounded_int(p.get(name),default=default,minimum=lo,maximum=hi,name=name)\n\ndef coords(value:Any,name:str,minimum:int,maximum:int)->list[list[float]]:\n if not isinstance(value,list) or not minimum<=len(value)<=maximum: raise ValueError(f'{name} size is invalid')\n out=[]\n for pair in value:\n if not isinstance(pair,list) or len(pair)!=2: raise ValueError(f'{name} coordinate is invalid')\n lon,lat=float(pair[0]),float(pair[1])\n if not -180<=lon<=180 or not -90<=lat<=90: raise ValueError(f'{name} coordinate is outside WGS84 bounds')\n out.append([lon,lat])\n return out\n\ndef build(operation:str,p:Mapping[str,Any])->Spec|dict[str,Any]:\n if operation=='catalog-capabilities':\n if p: raise ValueError('catalog-capabilities accepts no parameters')\n return {'provider':provider_row(CATALOG_PATH)}\n if operation=='china-local-open-data-catalog':\n if p: raise ValueError('china-local-open-data-catalog accepts no parameters')\n return {'portals':[\n {'name':'浙江·数据开放','url':'https://data.zjzwfw.gov.cn/dopServer/','coverage':'浙江省','access':'official portal; API/catalog access subject to portal registration'},\n {'name':'深圳市政府数据开放平台','url':'https://opendata.sz.gov.cn/','coverage':'深圳市','access':'official portal; machine-readable datasets and application interfaces vary by dataset'},\n {'name':'上海市公共数据开放平台','url':'https://data.sh.gov.cn/','coverage':'上海市','access':'official portal'},\n {'name':'北京市公共数据开放平台','url':'https://data.beijing.gov.cn/','coverage':'北京市','access':'official portal'},\n {'name':'广东省公共数据开放平台','url':'https://data.gd.gov.cn/','coverage':'广东省','access':'official portal'},\n {'name':'福建省公共数据资源统一开放平台','url':'https://data.fujian.gov.cn/','coverage':'福建省','access':'official portal; availability must be checked per dataset'},\n ],'note':'目录入口只做发现；未把登录、人工审批或人机验证包装为自动 API。'}\n if operation=='china-science-data-centers':\n if p: raise ValueError('china-science-data-centers accepts no parameters')\n return {'centers':[\n {'name':'国家地球系统科学数据中心','url':'https://www.geodata.cn','themes':['land','ecology','resources','environment']},\n {'name':'国家青藏高原科学数据中心','url':'https://data.tpdc.ac.cn','themes':['Tibetan Plateau','cryosphere','atmosphere','hydrology']},\n {'name':'国家林业和草原科学数据中心','url':'https://forestdata.cn','api':'key application; OGC WMTS map services'},\n {'name':'国家气象科学数据中心','url':'https://data.cma.cn','access':'registration and dataset-specific rules'},\n {'name':'国家地震科学数据中心','url':'https://data.earthquake.cn','access':'catalog and dataset-specific services'},\n {'name':'国家海洋科学数据中心','url':'https://mds.nmdis.org.cn','access':'catalog and dataset-specific services'},\n {'name':'国家生态科学数据中心','url':'https://www.cnern.org.cn','access':'catalog and station-network data'},\n {'name':'国家人口健康科学数据中心','url':'https://www.ncmi.cn','access':'registration/data-specific governance; no personal-data automation'},\n {'name':'国家基础地理信息中心公共地理数据','url':'https://www.webmap.cn','access':'public catalog and downloadable basic geographic resources'},\n ],'note':'仅收录公开科学数据目录；涉及个人健康、受控数据或逐项审批的数据不自动调用。'}\n if operation=='ilostat-dataflows':\n return Spec('GET','https://sdmx.ilo.org/rest/dataflow/all/all/latest',headers={'Accept':'application/vnd.sdmx.structure+json;version=2.0.0'})\n if operation=='unicef-dataflows':\n return Spec('GET','https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/dataflow/all/all/latest/',params=[('format','sdmx-json'),('detail','full'),('references','none')])\n if operation=='un-sdg-indicators':\n return Spec('GET','https://unstats.un.org/SDGAPI/v1/sdg/Indicator/List',params=[('includechildren','true' if p.get('include_children') else 'false')])\n if operation=='faostat-definitions':\n lang=text(p,'language',2) or 'en'; return Spec('GET',f'https://fenixservices.fao.org/faostat/api/v1/{lang}/Definitions')\n if operation=='worldpop-catalog':\n q=[]\n if p.get('alias'): q.append(('alias',text(p,'alias',80)))\n if p.get('iso3'): q.append(('iso3',text(p,'iso3',3)))\n return Spec('GET','https://www.worldpop.org/rest/data',params=q)\n if operation=='worldpop-services': return Spec('GET','https://api.worldpop.org/v1/services')\n if operation=='gbif-occurrences':\n q=[('limit',str(integer(p,'limit',20,1,100))),('offset',str(integer(p,'offset',0,0,100000)))]\n mapping={'scientific_name':'scientificName','country':'country','year':'year'}\n for s,t in mapping.items():\n if p.get(s): q.append((t,text(p,s,200)))\n if p.get('decimal_latitude') is not None and p.get('decimal_longitude') is not None:\n lat=float(p['decimal_latitude']); lon=float(p['decimal_longitude']); radius=float(p.get('radius_km') or 10)\n q.append(('geometry',f'POINT({lon} {lat})')); q.append(('geometry_srs','EPSG:4326'))\n q.append(('distanceFromCentroidInMeters',str(int(radius*1000))))\n return Spec('GET','https://api.gbif.org/v1/occurrence/search',params=q)\n if operation=='unhcr-population':\n q=[('limit',str(integer(p,'limit',20,1,100))),('page',str(integer(p,'page',1,1,1000)))]\n for s,t in {'year_from':'yearFrom','year_to':'yearTo','coa':'coa','coo':'coo'}.items():\n if p.get(s) not in (None,''): q.append((t,str(p[s])))\n return Spec('GET','https://api.unhcr.org/population/v1/population/',params=q)\n if operation=='reliefweb-reports':\n q=[('appname',secret('RELIEFWEB_APPNAME')),('limit',str(integer(p,'limit',20,1,50))),('offset',str(integer(p,'offset',0,0,10000))),('profile','list')]\n if p.get('query'): q.append(('query[value]',text(p,'query',300)))\n if p.get('country'): q.extend([('filter[field]','country.name'),('filter[value]',text(p,'country',100))])\n return Spec('GET','https://api.reliefweb.int/v2/reports',params=q,credential_mode='appname')\n if operation=='gleif-lei-search':\n q=[('filter[entity.legalName]',text(p,'query',200,True)),('page[size]',str(integer(p,'page_size',20,1,100))),('page[number]',str(integer(p,'page_number',1,1,1000)))]\n if p.get('country'): q.append(('filter[entity.legalAddress.country]',text(p,'country',2)))\n return Spec('GET','https://api.gleif.org/api/v1/lei-records',params=q)\n if operation=='usaspending-awards':\n start=text(p,'start_date',10,True); end=text(p,'end_date',10,True)\n body={'filters':{'keywords':p['keywords'],'time_period':[{'start_date':start,'end_date':end}]},'fields':['Award ID','Recipient Name','Award Amount','Awarding Agency','Start Date','End Date','Description'],'page':integer(p,'page',1,1,1000),'limit':integer(p,'limit',20,1,100),'subawards':False}\n if p.get('award_type_codes'): body['filters']['award_type_codes']=p['award_type_codes']\n return Spec('POST','https://api.usaspending.gov/api/v2/search/spending_by_award/',json_body=body)\n if operation=='openfda-drug-events':\n q=[('search',text(p,'search',500,True)),('limit',str(integer(p,'limit',20,1,100))),('skip',str(integer(p,'skip',0,0,25000)))]\n key=secret('OPENFDA_API_KEY',False)\n if key: q.append(('api_key',key))\n return Spec('GET','https://api.fda.gov/drug/event.json',params=q,credential_mode='optional-api-key' if key else 'none')\n if operation=='eurostat-data':\n flow=text(p,'dataflow',80,True); key=text(p,'key',300,True)\n q=[('format','csvfile')]\n if p.get('start_period'): q.append(('startPeriod',text(p,'start_period',20)))\n if p.get('end_period'): q.append(('endPeriod',text(p,'end_period',20)))\n return Spec('GET',f'https://ec.europa.eu/eurostat/api/dissemination/sdmx/3.0/data/ESTAT/{flow}/1.0/{key}',params=q,response_kind='text')\n if operation=='our-world-in-data-series':\n return Spec('GET',f\"https://ourworldindata.org/grapher/{text(p,'slug',120,True)}.csv\",response_kind='text')\n if operation=='hdx-hapi-metadata':\n app=secret('HDX_HAPI_APP_IDENTIFIER'); return Spec('GET','https://hapi.humdata.org/api/v2/metadata/',params=[('app_identifier',app)],credential_mode='app-identifier')\n if operation=='iati-activities':\n key=secret('IATI_API_KEY'); body={'q':text(p,'query',500,True),'rows':integer(p,'rows',20,1,100),'start':integer(p,'start',0,0,10000)}\n return Spec('POST','https://api.iatistandard.org/datastore/activity/select',json_body=body,headers={'Ocp-Apim-Subscription-Key':key},credential_mode='api-key')\n if operation=='companies-house-search':\n key=secret('COMPANIES_HOUSE_API_KEY'); q=[('q',text(p,'query',200,True)),('items_per_page',str(integer(p,'items_per_page',20,1,100))),('start_index',str(integer(p,'start_index',0,0,10000)))]\n return Spec('GET','https://api.company-information.service.gov.uk/search/companies',params=q,auth=(key,''),credential_mode='basic-api-key')\n if operation=='sam-opportunities':\n q=[('api_key',secret('SAM_GOV_API_KEY')),('postedFrom',text(p,'posted_from',10,True)),('postedTo',text(p,'posted_to',10,True)),('limit',str(integer(p,'limit',20,1,100))),('offset',str(integer(p,'offset',0,0,1000)))]\n if p.get('keywords'): q.append(('q',text(p,'keywords',200)))\n return Spec('GET','https://api.sam.gov/opportunities/v2/search',params=q,credential_mode='api-key')\n if operation=='openaq-locations':\n key=secret('OPENAQ_API_KEY'); q=[('limit',str(integer(p,'limit',20,1,100))),('page',str(integer(p,'page',1,1,1000)))]\n for s in ('country','city','coordinates','radius'):\n if p.get(s) not in (None,''): q.append((s,str(p[s])))\n return Spec('GET','https://api.openaq.org/v3/locations',params=q,headers={'X-API-Key':key},credential_mode='api-key')\n if operation=='usgs-earthquakes':\n q=[('format','geojson'),('limit',str(integer(p,'limit',200,1,2000))),('orderby','time')]\n for s,t in {'start_time':'starttime','end_time':'endtime','min_magnitude':'minmagnitude','latitude':'latitude','longitude':'longitude','max_radius_km':'maxradiuskm'}.items():\n if p.get(s) not in (None,''): q.append((t,str(p[s])))\n return Spec('GET','https://earthquake.usgs.gov/fdsnws/event/1/query',params=q)\n if operation=='overpass-query':\n q=text(p,'query',8000,True)\n low=q.lower()\n if '[out:json]' not in low or any(x in low for x in ('[out:xml]','adiff','delete','make area','{{')): raise ValueError('overpass query must be bounded read-only JSON QL')\n timeout=integer(p,'timeout_seconds',20,1,25)\n if '[timeout:' not in low: q=f'[out:json][timeout:{timeout}];'+q.replace('[out:json];','',1)\n return Spec('POST','https://overpass-api.de/api/interpreter',data_body={'data':q},response_kind='json')\n if operation=='geonames-search':\n q=[('q',text(p,'query',200,True)),('username',secret('GEONAMES_USERNAME')),('maxRows',str(integer(p,'max_rows',20,1,100))),('startRow',str(integer(p,'start_row',0,0,10000))),('style','FULL')]\n if p.get('country'): q.append(('country',text(p,'country',2)))\n if p.get('feature_class'): q.append(('featureClass',text(p,'feature_class',1)))\n return Spec('GET','https://secure.geonames.org/searchJSON',params=q,credential_mode='username')\n if operation.startswith('openrouteservice-'):\n key=secret('OPENROUTESERVICE_API_KEY'); headers={'Authorization':key,'Content-Type':'application/json'}\n if operation=='openrouteservice-geocode':\n q=[('text',text(p,'text',300,True)),('size',str(integer(p,'size',10,1,40)))]\n if p.get('country'): q.append(('boundary.country',text(p,'country',2).lower()))\n if p.get('focus_point_lat') is not None and p.get('focus_point_lon') is not None:\n q.extend([('focus.point.lat',str(float(p['focus_point_lat']))),('focus.point.lon',str(float(p['focus_point_lon'])))])\n return Spec('GET','https://api.heigit.org/pelias/v1/search',params=q,headers={'Authorization':key},credential_mode='api-key')\n profile=text(p,'profile',30,True)\n if operation=='openrouteservice-directions':\n body={'coordinates':coords(p.get('coordinates'),'coordinates',2,50)}\n for name in ('preference','units','language','instructions'):\n if p.get(name) is not None: body[name]=p[name]\n return Spec('POST',f'https://api.heigit.org/openrouteservice/v2/directions/{profile}/geojson',json_body=body,headers=headers,credential_mode='api-key')\n if operation=='openrouteservice-matrix':\n body={'locations':coords(p.get('locations'),'locations',2,50),'metrics':p.get('metrics') or ['duration']}\n if p.get('units'): body['units']=p['units']\n return Spec('POST',f'https://api.heigit.org/openrouteservice/v2/matrix/{profile}',json_body=body,headers=headers,credential_mode='api-key')\n body={'locations':coords(p.get('locations'),'locations',1,5),'range':p.get('range')}\n if not isinstance(body['range'],list) or not 1<=len(body['range'])<=10: raise ValueError('range is invalid')\n body['range_type']=p.get('range_type') or 'time'\n if p.get('units'): body['units']=p['units']\n return Spec('POST',f'https://api.heigit.org/openrouteservice/v2/isochrones/{profile}',json_body=body,headers=headers,credential_mode='api-key')\n if operation=='opentopography-globaldem':\n south,north,west,east=map(float,(p['south'],p['north'],p['west'],p['east']))\n if not south<north or not west<east or (north-south)*(east-west)>25: raise ValueError('DEM bounding box must be ordered and <=25 square degrees')\n q=[('demtype',text(p,'dem_type',20,True)),('south',str(south)),('north',str(north)),('west',str(west)),('east',str(east)),('outputFormat',str(p.get('output_format') or 'GTiff')),('API_Key',secret('OPENTOPOGRAPHY_API_KEY'))]\n return Spec('GET','https://portal.opentopography.org/API/globaldem',params=q,credential_mode='api-key',response_kind='binary')\n if operation=='geoboundaries-release':\n product=str(p.get('product') or 'gbOpen'); iso=text(p,'iso3',3,True); adm=text(p,'admin_level',4,True)\n return Spec('GET',f'https://www.geoboundaries.org/api/current/{product}/{iso}/{adm}/')\n if operation=='soilgrids-wcs-capabilities':\n prop=text(p,'property',20,True); return Spec('GET','https://maps.isric.org/mapserv',params=[('map',f'/map/{prop}.map'),('SERVICE','WCS'),('VERSION','2.0.1'),('REQUEST','GetCapabilities')],response_kind='text')\n if operation=='global-fishing-watch-vessels':\n key=secret('GLOBAL_FISHING_WATCH_API_TOKEN'); q=[('query',text(p,'query',100,True)),('limit',str(integer(p,'limit',20,1,50)))]\n return Spec('GET','https://gateway.api.globalfishingwatch.org/v3/vessels/search',params=q,headers={'Authorization':f'Bearer {key}'},credential_mode='bearer-token')\n if operation=='opencharge-map-poi':\n q=[('latitude',str(float(p['latitude']))),('longitude',str(float(p['longitude']))),('distance',str(float(p.get('distance_km') or 25))),('distanceunit','KM'),('maxresults',str(integer(p,'max_results',50,1,100))),('compact','true'),('verbose','false')]\n key=secret('OPENCHARGEMAP_API_KEY',False); headers={'X-API-Key':key} if key else {}\n return Spec('GET','https://api.openchargemap.io/v3/poi/',params=q,headers=headers,credential_mode='optional-api-key' if key else 'none')\n if operation=='transitland-routes':\n q=[('limit',str(integer(p,'limit',20,1,100)))]\n if p.get('bbox'): q.append(('bbox',text(p,'bbox',100)))\n if p.get('operator_onestop_id'): q.append(('operator_onestop_id',text(p,'operator_onestop_id',100)))\n key=secret('TRANSITLAND_API_KEY',False)\n if key: q.append(('apikey',key))\n return Spec('GET','https://transit.land/api/v2/rest/routes',params=q,credential_mode='optional-api-key' if key else 'none')\n raise ValueError(f'unsupported operation: {operation}')\n\ndef row_count(payload:Any)->int:\n if isinstance(payload,list): return len(payload)\n if isinstance(payload,Mapping):\n for key in ('data','results','features','entries','items','records','geonames','response','portals','centers'):\n v=payload.get(key)\n if isinstance(v,list): return len(v)\n if isinstance(v,Mapping):\n for sub in ('data','results','items','records'):\n sv=v.get(sub)\n if isinstance(sv,list): return len(sv)\n return 1\n return 0\n\ndef execute(ticket_path:Path,output_dir:Path)->int:\n output_dir.mkdir(parents=True,exist_ok=True)\n ticket=load_json(ticket_path); validate_ticket(ticket,schema_path=SCHEMA_PATH,catalog_path=CATALOG_PATH)\n operation=str(ticket['operation']); p=dict(ticket.get('parameters') or {}); acc=dict(ticket['acceptance'])\n timeout=bounded_int(acc.get('timeout_seconds'),default=30,minimum=5,maximum=120,name='timeout_seconds')\n max_bytes=bounded_int(acc.get('max_response_bytes'),default=5000000,minimum=1024,maximum=20000000,name='max_response_bytes')\n started_at,started_perf=utc_now(),time.perf_counter(); status='INTEL_PUBLIC_DATA_FAILED'; failure=None; snapshot=None\n metadata={'upstream_called':False,'requests_per_ticket_max':1,'automatic_retry':False,'automatic_pagination':False,'secret_values_exposed':False,'operation':operation}\n try:\n spec=build(operation,p)\n if isinstance(spec,dict): snapshot={'provider':'public-data-geospatial','operation':operation,'row_count':row_count(spec),'data':spec}\n else:\n parsed=urlparse(spec.url)\n if parsed.scheme!='https' or not parsed.hostname: raise RuntimeError('non-HTTPS or invalid fixed endpoint')\n headers={'Accept':'application/json, application/xml;q=0.9, text/csv;q=0.8, text/plain;q=0.7','User-Agent':'intelligence-center-public-data/1',**(spec.headers or {})}\n kwargs={'params':spec.params,'headers':headers,'timeout':timeout,'allow_redirects':False}\n if spec.auth: kwargs['auth']=spec.auth\n if spec.method=='POST':\n if spec.json_body is not None: kwargs['json']=spec.json_body\n if spec.data_body is not None: kwargs['data']=spec.data_body\n response=requests.post(spec.url,**kwargs)\n else: response=requests.get(spec.url,**kwargs)\n raw=bytes(response.content or b'')\n metadata.update({'upstream_called':True,'api_host':parsed.hostname,'request_path':parsed.path,'http_status':response.status_code,'content_type':response.headers.get('Content-Type',''),'response_bytes_raw':len(raw),'credential_mode':spec.credential_mode})\n if len(raw)>max_bytes: raise RuntimeError(f'response exceeds acceptance.max_response_bytes={max_bytes}')\n if not response.ok: raise RuntimeError(f'upstream HTTP {response.status_code}: {raw[:1000].decode(\"utf-8\",errors=\"replace\")}')\n if spec.response_kind=='binary':\n stored=raw; (output_dir/'response.bin').write_bytes(stored); data={'binary':True,'content_type':response.headers.get('Content-Type',''),'bytes':len(stored)}\n elif spec.response_kind=='text':\n value=raw.decode('utf-8',errors='replace'); stored=(value if value.endswith('\\n') else value+'\\n').encode(); (output_dir/'response.txt').write_bytes(stored); data={'text':value[:200000],'truncated_for_snapshot':len(value)>200000}\n else:\n try: data=response.json()\n except ValueError:\n value=raw.decode('utf-8',errors='replace'); data={'text':value[:200000],'truncated_for_snapshot':len(value)>200000}\n stored=(json.dumps(data,ensure_ascii=False,indent=2,allow_nan=False)+'\\n').encode(); (output_dir/'response.json').write_bytes(stored)\n if len(stored)>max_bytes: raise RuntimeError('sanitized response exceeds max_response_bytes')\n rows=row_count(data); snapshot={'provider':'public-data-geospatial','operation':operation,'row_count':rows,'data':data}\n metadata.update({'response_bytes':len(stored),'response_sha256':bytes_sha(stored),'row_count':rows})\n status='INTEL_PUBLIC_DATA_COMPLETED'\n except Exception as exc: failure={'type':type(exc).__name__,'message':str(exc)[:2000]}\n return finish_execution(ticket=ticket,output_dir=output_dir,status=status,snapshot=snapshot,metadata=metadata,failure=failure,started_at=started_at,started_perf=started_perf,schema_prefix='public-data-geospatial')\n\nif __name__=='__main__':\n raise SystemExit(run_cli(execute=execute,ticket_prefix='[intel-public-data]',schema_path=SCHEMA_PATH,catalog_path=CATALOG_PATH,status_schema='public-data-geospatial-ticket-status-v1',display_name='Public Data & Geospatial'))\n"}, "93522ec10744ed9bb9e03592a1b01dc37d7db67d": {"t": 1785809908.7869403, "n": 22436, "h": "cbb9cc0459e7b06b", "c": "#!/usr/bin/env python3\n\"\"\"Bounded, fixed-host public-data and geospatial provider.\"\"\"\nfrom __future__ import annotations\nimport json\nimport os\nimport sys\nimport time\nfrom pathlib import Path\nfrom typing import Any, Mapping\nfrom urllib.parse import urlparse\nimport requests\n\nHERE=Path(__file__).resolve().parent\nsys.path.insert(0,str(HERE.parent))\nfrom managed_provider_runtime import bounded_int, bytes_sha, finish_execution, load_json, provider_row, run_cli, utc_now, validate_ticket\nSCHEMA_PATH=HERE/'ticket.schema.json'; CATALOG_PATH=HERE/'provider-catalog.json'\n\nclass Spec:\n def __init__(self, method:str, url:str, params=None, json_body=None, data_body=None, headers=None, auth=None, credential_mode='none', response_kind='json'):\n self.method=method; self.url=url; self.params=params; self.json_body=json_body\n self.data_body=data_body; self.headers=headers; self.auth=auth\n self.credential_mode=credential_mode; self.response_kind=response_kind\n\ndef text(p:Mapping[str,Any],name:str,maximum:int=300,required:bool=False)->str:\n value=str(p.get(name) or '').strip()\n if required and not value: raise ValueError(f'{name} is required')\n if len(value)>maximum or any(ord(ch)<32 for ch in value): raise ValueError(f'{name} is invalid')\n return value\n\ndef secret(name:str,required:bool=True)->str:\n value=str(os.environ.get(name) or '').strip()\n if required and not value: raise RuntimeError(f'missing required backend secret or variable: {name}')\n return value\n\ndef integer(p:Mapping[str,Any],name:str,default:int,lo:int,hi:int)->int:\n return bounded_int(p.get(name),default=default,minimum=lo,maximum=hi,name=name)\n\ndef coords(value:Any,name:str,minimum:int,maximum:int)->list[list[float]]:\n if not isinstance(value,list) or not minimum<=len(value)<=maximum: raise ValueError(f'{name} size is invalid')\n out=[]\n for pair in value:\n if not isinstance(pair,list) or len(pair)!=2: raise ValueError(f'{name} coordinate is invalid')\n lon,lat=float(pair[0]),float(pair[1])\n if not -180<=lon<=180 or not -90<=lat<=90: raise ValueError(f'{name} coordinate is outside WGS84 bounds')\n out.append([lon,lat])\n return out\n\ndef build(operation:str,p:Mapping[str,Any])->Spec|dict[str,Any]:\n if operation=='catalog-capabilities':\n if p: raise ValueError('catalog-capabilities accepts no parameters')\n return {'provider':provider_row(CATALOG_PATH)}\n if operation=='china-local-open-data-catalog':\n if p: raise ValueError('china-local-open-data-catalog accepts no parameters')\n return {'portals':[\n {'name':'浙江·数据开放','url':'https://data.zjzwfw.gov.cn/dopServer/','coverage':'浙江省','access':'official portal; API/catalog access subject to portal registration'},\n {'name':'深圳市政府数据开放平台','url':'https://opendata.sz.gov.cn/','coverage':'深圳市','access':'official portal; machine-readable datasets and application interfaces vary by dataset'},\n {'name':'上海市公共数据开放平台','url':'https://data.sh.gov.cn/','coverage':'上海市','access':'official portal'},\n {'name':'北京市公共数据开放平台','url':'https://data.beijing.gov.cn/','coverage':'北京市','access':'official portal'},\n {'name':'广东省公共数据开放平台','url':'https://data.gd.gov.cn/','coverage':'广东省','access':'official portal'},\n {'name':'福建省公共数据资源统一开放平台','url':'https://data.fujian.gov.cn/','coverage':'福建省','access':'official portal; availability must be checked per dataset'},\n ],'note':'目录入口只做发现；未把登录、人工审批或人机验证包装为自动 API。'}\n if operation=='china-science-data-centers':\n if p: raise ValueError('china-science-data-centers accepts no parameters')\n return {'centers':[\n {'name':'国家地球系统科学数据中心','url':'https://www.geodata.cn','themes':['land','ecology','resources','environment']},\n {'name':'国家青藏高原科学数据中心','url':'https://data.tpdc.ac.cn','themes':['Tibetan Plateau','cryosphere','atmosphere','hydrology']},\n {'name':'国家林业和草原科学数据中心','url':'https://forestdata.cn','api':'key application; OGC WMTS map services'},\n {'name':'国家气象科学数据中心','url':'https://data.cma.cn','access':'registration and dataset-specific rules'},\n {'name':'国家地震科学数据中心','url':'https://data.earthquake.cn','access':'catalog and dataset-specific services'},\n {'name':'国家海洋科学数据中心','url':'https://mds.nmdis.org.cn','access':'catalog and dataset-specific services'},\n {'name':'国家生态科学数据中心','url':'https://www.cnern.org.cn','access':'catalog and station-network data'},\n {'name':'国家人口健康科学数据中心','url':'https://www.ncmi.cn','access':'registration/data-specific governance; no personal-data automation'},\n {'name':'国家基础地理信息中心公共地理数据','url':'https://www.webmap.cn','access':'public catalog and downloadable basic geographic resources'},\n ],'note':'仅收录公开科学数据目录；涉及个人健康、受控数据或逐项审批的数据不自动调用。'}\n if operation=='ilostat-dataflows':\n return Spec('GET','https://sdmx.ilo.org/rest/dataflow/all/all/latest',headers={'Accept':'application/vnd.sdmx.structure+json;version=2.0.0'})\n if operation=='unicef-dataflows':\n return Spec('GET','https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/dataflow/all/all/latest/',params=[('format','sdmx-json'),('detail','full'),('references','none')])\n if operation=='un-sdg-indicators':\n return Spec('GET','https://unstats.un.org/SDGAPI/v1/sdg/Indicator/List',params=[('includechildren','true' if p.get('include_children') else 'false')])\n if operation=='faostat-definitions':\n lang=text(p,'language',2) or 'en'; return Spec('GET',f'https://fenixservices.fao.org/faostat/api/v1/{lang}/Definitions')\n if operation=='worldpop-catalog':\n q=[]\n if p.get('alias'): q.append(('alias',text(p,'alias',80)))\n if p.get('iso3'): q.append(('iso3',text(p,'iso3',3)))\n return Spec('GET','https://www.worldpop.org/rest/data',params=q)\n if operation=='worldpop-services': return Spec('GET','https://api.worldpop.org/v1/services')\n if operation=='gbif-occurrences':\n q=[('limit',str(integer(p,'limit',20,1,100))),('offset',str(integer(p,'offset',0,0,100000)))]\n mapping={'scientific_name':'scientificName','country':'country','year':'year'}\n for s,t in mapping.items():\n if p.get(s): q.append((t,text(p,s,200)))\n if p.get('decimal_latitude') is not None and p.get('decimal_longitude') is not None:\n lat=float(p['decimal_latitude']); lon=float(p['decimal_longitude']); radius=float(p.get('radius_km') or 10)\n q.append(('geometry',f'POINT({lon} {lat})')); q.append(('geometry_srs','EPSG:4326'))\n q.append(('distanceFromCentroidInMeters',str(int(radius*1000))))\n return Spec('GET','https://api.gbif.org/v1/occurrence/search',params=q)\n if operation=='unhcr-population':\n q=[('limit',str(integer(p,'limit',20,1,100))),('page',str(integer(p,'page',1,1,1000)))]\n for s,t in {'year_from':'yearFrom','year_to':'yearTo','coa':'coa','coo':'coo'}.items():\n if p.get(s) not in (None,''): q.append((t,str(p[s])))\n return Spec('GET','https://api.unhcr.org/population/v1/population/',params=q)\n if operation=='reliefweb-reports':\n q=[('appname',secret('RELIEFWEB_APPNAME')),('limit',str(integer(p,'limit',20,1,50))),('offset',str(integer(p,'offset',0,0,10000))),('profile','list')]\n if p.get('query'): q.append(('query[value]',text(p,'query',300)))\n if p.get('country'): q.extend([('filter[field]','country.name'),('filter[value]',text(p,'country',100))])\n return Spec('GET','https://api.reliefweb.int/v2/reports',params=q,credential_mode='appname')\n if operation=='gleif-lei-search':\n q=[('filter[entity.legalName]',text(p,'query',200,True)),('page[size]',str(integer(p,'page_size',20,1,100))),('page[number]',str(integer(p,'page_number',1,1,1000)))]\n if p.get('country'): q.append(('filter[entity.legalAddress.country]',text(p,'country',2)))\n return Spec('GET','https://api.gleif.org/api/v1/lei-records',params=q)\n if operation=='usaspending-awards':\n start=text(p,'start_date',10,True); end=text(p,'end_date',10,True)\n body={'filters':{'keywords':p['keywords'],'time_period':[{'start_date':start,'end_date':end}]},'fields':['Award ID','Recipient Name','Award Amount','Awarding Agency','Start Date','End Date','Description'],'page':integer(p,'page',1,1,1000),'limit':integer(p,'limit',20,1,100),'subawards':False}\n if p.get('award_type_codes'): body['filters']['award_type_codes']=p['award_type_codes']\n return Spec('POST','https://api.usaspending.gov/api/v2/search/spending_by_award/',json_body=body)\n if operation=='openfda-drug-events':\n q=[('search',text(p,'search',500,True)),('limit',str(integer(p,'limit',20,1,100))),('skip',str(integer(p,'skip',0,0,25000)))]\n key=secret('OPENFDA_API_KEY',False)\n if key: q.append(('api_key',key))\n return Spec('GET','https://api.fda.gov/drug/event.json',params=q,credential_mode='optional-api-key' if key else 'none')\n if operation=='eurostat-data':\n flow=text(p,'dataflow',80,True); key=text(p,'key',300,True)\n q=[('format','csvfile')]\n if p.get('start_period'): q.append(('startPeriod',text(p,'start_period',20)))\n if p.get('end_period'): q.append(('endPeriod',text(p,'end_period',20)))\n return Spec('GET',f'https://ec.europa.eu/eurostat/api/dissemination/sdmx/3.0/data/ESTAT/{flow}/1.0/{key}',params=q,response_kind='text')\n if operation=='our-world-in-data-series':\n return Spec('GET',f\"https://ourworldindata.org/grapher/{text(p,'slug',120,True)}.csv\",response_kind='text')\n if operation=='hdx-hapi-metadata':\n app=secret('HDX_HAPI_APP_IDENTIFIER'); return Spec('GET','https://hapi.humdata.org/api/v2/metadata/',params=[('app_identifier',app)],credential_mode='app-identifier')\n if operation=='iati-activities':\n key=secret('IATI_API_KEY'); body={'q':text(p,'query',500,True),'rows':integer(p,'rows',20,1,100),'start':integer(p,'start',0,0,10000)}\n return Spec('POST','https://api.iatistandard.org/datastore/activity/select',json_body=body,headers={'Ocp-Apim-Subscription-Key':key},credential_mode='api-key')\n if operation=='companies-house-search':\n key=secret('COMPANIES_HOUSE_API_KEY'); q=[('q',text(p,'query',200,True)),('items_per_page',str(integer(p,'items_per_page',20,1,100))),('start_index',str(integer(p,'start_index',0,0,10000)))]\n return Spec('GET','https://api.company-information.service.gov.uk/search/companies',params=q,auth=(key,''),credential_mode='basic-api-key')\n if operation=='sam-opportunities':\n q=[('api_key',secret('SAM_GOV_API_KEY')),('postedFrom',text(p,'posted_from',10,True)),('postedTo',text(p,'posted_to',10,True)),('limit',str(integer(p,'limit',20,1,100))),('offset',str(integer(p,'offset',0,0,1000)))]\n if p.get('keywords'): q.append(('q',text(p,'keywords',200)))\n return Spec('GET','https://api.sam.gov/opportunities/v2/search',params=q,credential_mode='api-key')\n if operation=='openaq-locations':\n key=secret('OPENAQ_API_KEY'); q=[('limit',str(integer(p,'limit',20,1,100))),('page',str(integer(p,'page',1,1,1000)))]\n for s in ('country','city','coordinates','radius'):\n if p.get(s) not in (None,''): q.append((s,str(p[s])))\n return Spec('GET','https://api.openaq.org/v3/locations',params=q,headers={'X-API-Key':key},credential_mode='api-key')\n if operation=='usgs-earthquakes':\n q=[('format','geojson'),('limit',str(integer(p,'limit',200,1,2000))),('orderby','time')]\n for s,t in {'start_time':'starttime','end_time':'endtime','min_magnitude':'minmagnitude','latitude':'latitude','longitude':'longitude','max_radius_km':'maxradiuskm'}.items():\n if p.get(s) not in (None,''): q.append((t,str(p[s])))\n return Spec('GET','https://earthquake.usgs.gov/fdsnws/event/1/query',params=q)\n if operation=='overpass-query':\n q=text(p,'query',8000,True)\n low=q.lower()\n if '[out:json]' not in low or any(x in low for x in ('[out:xml]','adiff','delete','make area','{{')): raise ValueError('overpass query must be bounded read-only JSON QL')\n timeout=integer(p,'timeout_seconds',20,1,25)\n if '[timeout:' not in low: q=f'[out:json][timeout:{timeout}];'+q.replace('[out:json];','',1)\n return Spec('POST','https://overpass-api.de/api/interpreter',data_body={'data':q},response_kind='json')\n if operation=='geonames-search':\n q=[('q',text(p,'query',200,True)),('username',secret('GEONAMES_USERNAME')),('maxRows',str(integer(p,'max_rows',20,1,100))),('startRow',str(integer(p,'start_row',0,0,10000))),('style','FULL')]\n if p.get('country'): q.append(('country',text(p,'country',2)))\n if p.get('feature_class'): q.append(('featureClass',text(p,'feature_class',1)))\n return Spec('GET','https://secure.geonames.org/searchJSON',params=q,credential_mode='username')\n if operation.startswith('openrouteservice-'):\n key=secret('OPENROUTESERVICE_API_KEY'); headers={'Authorization':key,'Content-Type':'application/json'}\n if operation=='openrouteservice-geocode':\n q=[('text',text(p,'text',300,True)),('size',str(integer(p,'size',10,1,40)))]\n if p.get('country'): q.append(('boundary.country',text(p,'country',2).lower()))\n if p.get('focus_point_lat') is not None and p.get('focus_point_lon') is not None:\n q.extend([('focus.point.lat',str(float(p['focus_point_lat']))),('focus.point.lon',str(float(p['focus_point_lon'])))])\n return Spec('GET','https://api.heigit.org/pelias/v1/search',params=q,headers={'Authorization':key},credential_mode='api-key')\n profile=text(p,'profile',30,True)\n if operation=='openrouteservice-directions':\n body={'coordinates':coords(p.get('coordinates'),'coordinates',2,50)}\n for name in ('preference','units','language','instructions'):\n if p.get(name) is not None: body[name]=p[name]\n return Spec('POST',f'https://api.heigit.org/openrouteservice/v2/directions/{profile}/geojson',json_body=body,headers=headers,credential_mode='api-key')\n if operation=='openrouteservice-matrix':\n body={'locations':coords(p.get('locations'),'locations',2,50),'metrics':p.get('metrics') or ['duration']}\n if p.get('units'): body['units']=p['units']\n return Spec('POST',f'https://api.heigit.org/openrouteservice/v2/matrix/{profile}',json_body=body,headers=headers,credential_mode='api-key')\n body={'locations':coords(p.get('locations'),'locations',1,5),'range':p.get('range')}\n if not isinstance(body['range'],list) or not 1<=len(body['range'])<=10: raise ValueError('range is invalid')\n body['range_type']=p.get('range_type') or 'time'\n if p.get('units'): body['units']=p['units']\n return Spec('POST',f'https://api.heigit.org/openrouteservice/v2/isochrones/{profile}',json_body=body,headers=headers,credential_mode='api-key')\n if operation=='opentopography-globaldem':\n south,north,west,east=map(float,(p['south'],p['north'],p['west'],p['east']))\n if not south<north or not west<east or (north-south)*(east-west)>25: raise ValueError('DEM bounding box must be ordered and <=25 square degrees')\n q=[('demtype',text(p,'dem_type',20,True)),('south',str(south)),('north',str(north)),('west',str(west)),('east',str(east)),('outputFormat',str(p.get('output_format') or 'GTiff')),('API_Key',secret('OPENTOPOGRAPHY_API_KEY'))]\n return Spec('GET','https://portal.opentopography.org/API/globaldem',params=q,credential_mode='api-key',response_kind='binary')\n if operation=='geoboundaries-release':\n product=str(p.get('product') or 'gbOpen'); iso=text(p,'iso3',3,True); adm=text(p,'admin_level',4,True)\n return Spec('GET',f'https://www.geoboundaries.org/api/current/{product}/{iso}/{adm}/')\n if operation=='soilgrids-wcs-capabilities':\n prop=text(p,'property',20,True); return Spec('GET','https://maps.isric.org/mapserv',params=[('map',f'/map/{prop}.map'),('SERVICE','WCS'),('VERSION','2.0.1'),('REQUEST','GetCapabilities')],response_kind='text')\n if operation=='global-fishing-watch-vessels':\n key=secret('GLOBAL_FISHING_WATCH_API_TOKEN'); q=[('query',text(p,'query',100,True)),('limit',str(integer(p,'limit',20,1,50)))]\n return Spec('GET','https://gateway.api.globalfishingwatch.org/v3/vessels/search',params=q,headers={'Authorization':f'Bearer {key}'},credential_mode='bearer-token')\n if operation=='opencharge-map-poi':\n q=[('latitude',str(float(p['latitude']))),('longitude',str(float(p['longitude']))),('distance',str(float(p.get('distance_km') or 25))),('distanceunit','KM'),('maxresults',str(integer(p,'max_results',50,1,100))),('compact','true'),('verbose','false')]\n key=secret('OPENCHARGEMAP_API_KEY',False); headers={'X-API-Key':key} if key else {}\n return Spec('GET','https://api.openchargemap.io/v3/poi/',params=q,headers=headers,credential_mode='optional-api-key' if key else 'none')\n if operation=='transitland-routes':\n q=[('limit',str(integer(p,'limit',20,1,100)))]\n if p.get('bbox'): q.append(('bbox',text(p,'bbox',100)))\n if p.get('operator_onestop_id'): q.append(('operator_onestop_id',text(p,'operator_onestop_id',100)))\n key=secret('TRANSITLAND_API_KEY',False)\n if key: q.append(('apikey',key))\n return Spec('GET','https://transit.land/api/v2/rest/routes',params=q,credential_mode='optional-api-key' if key else 'none')\n raise ValueError(f'unsupported operation: {operation}')\n\ndef row_count(payload:Any)->int:\n if isinstance(payload,list): return len(payload)\n if isinstance(payload,Mapping):\n for key in ('data','results','features','entries','items','records','geonames','response','portals','centers'):\n v=payload.get(key)\n if isinstance(v,list): return len(v)\n if isinstance(v,Mapping):\n for sub in ('data','results','items','records'):\n sv=v.get(sub)\n if isinstance(sv,list): return len(sv)\n return 1\n return 0\n\ndef execute(ticket_path:Path,output_dir:Path)->int:\n output_dir.mkdir(parents=True,exist_ok=True)\n ticket=load_json(ticket_path); validate_ticket(ticket,schema_path=SCHEMA_PATH,catalog_path=CATALOG_PATH)\n operation=str(ticket['operation']); p=dict(ticket.get('parameters') or {}); acc=dict(ticket['acceptance'])\n timeout=bounded_int(acc.get('timeout_seconds'),default=30,minimum=5,maximum=120,name='timeout_seconds')\n max_bytes=bounded_int(acc.get('max_response_bytes'),default=5000000,minimum=1024,maximum=20000000,name='max_response_bytes')\n started_at,started_perf=utc_now(),time.perf_counter(); status='INTEL_PUBLIC_DATA_FAILED'; failure=None; snapshot=None\n metadata={'upstream_called':False,'requests_per_ticket_max':1,'automatic_retry':False,'automatic_pagination':False,'secret_values_exposed':False,'operation':operation}\n try:\n spec=build(operation,p)\n if isinstance(spec,dict): snapshot={'provider':'public-data-geospatial','operation':operation,'row_count':row_count(spec),'data':spec}\n else:\n parsed=urlparse(spec.url)\n if parsed.scheme!='https' or not parsed.hostname: raise RuntimeError('non-HTTPS or invalid fixed endpoint')\n headers={'Accept':'application/json, application/xml;q=0.9, text/csv;q=0.8, text/plain;q=0.7','User-Agent':'intelligence-center-public-data/1',**(spec.headers or {})}\n kwargs={'params':spec.params,'headers':headers,'timeout':timeout,'allow_redirects':False}\n if spec.auth: kwargs['auth']=spec.auth\n if spec.method=='POST':\n if spec.json_body is not None: kwargs['json']=spec.json_body\n if spec.data_body is not None: kwargs['data']=spec.data_body\n response=requests.post(spec.url,**kwargs)\n else: response=requests.get(spec.url,**kwargs)\n raw=bytes(response.content or b'')\n metadata.update({'upstream_called':True,'api_host':parsed.hostname,'request_path':parsed.path,'http_status':response.status_code,'content_type':response.headers.get('Content-Type',''),'response_bytes_raw':len(raw),'credential_mode':spec.credential_mode})\n if len(raw)>max_bytes: raise RuntimeError(f'response exceeds acceptance.max_response_bytes={max_bytes}')\n if not response.ok: raise RuntimeError(f'upstream HTTP {response.status_code}: {raw[:1000].decode(\"utf-8\",errors=\"replace\")}')\n if spec.response_kind=='binary':\n stored=raw; (output_dir/'response.bin').write_bytes(stored); data={'binary':True,'content_type':response.headers.get('Content-Type',''),'bytes':len(stored)}\n elif spec.response_kind=='text':\n value=raw.decode('utf-8',errors='replace'); stored=(value if value.endswith('\\n') else value+'\\n').encode(); (output_dir/'response.txt').write_bytes(stored); data={'text':value[:200000],'truncated_for_snapshot':len(value)>200000}\n else:\n try: data=response.json()\n except ValueError:\n value=raw.decode('utf-8',errors='replace'); data={'text':value[:200000],'truncated_for_snapshot':len(value)>200000}\n stored=(json.dumps(data,ensure_ascii=False,indent=2,allow_nan=False)+'\\n').encode(); (output_dir/'response.json').write_bytes(stored)\n if len(stored)>max_bytes: raise RuntimeError('sanitized response exceeds max_response_bytes')\n rows=row_count(data); snapshot={'provider':'public-data-geospatial','operation':operation,'row_count':rows,'data':data}\n metadata.update({'response_bytes':len(stored),'response_sha256':bytes_sha(stored),'row_count':rows})\n status='INTEL_PUBLIC_DATA_COMPLETED'\n except Exception as exc: failure={'type':type(exc).__name__,'message':str(exc)[:2000]}\n return finish_execution(ticket=ticket,output_dir=output_dir,status=status,snapshot=snapshot,metadata=metadata,failure=failure,started_at=started_at,started_perf=started_perf,schema_prefix='public-data-geospatial')\n\nif __name__=='__main__':\n raise SystemExit(run_cli(execute=execute,ticket_prefix='[intel-public-data]',schema_path=SCHEMA_PATH,catalog_path=CATALOG_PATH,status_schema='public-data-geospatial-ticket-status-v1',display_name='Public Data & Geospatial'))\n"}}}} |