Add country metadata and source quality signals for immigration research.
Browse files
apis/country_profile.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Country profile helpers for the Borderless agent."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from apis.official_sources import hints_for_country
|
| 8 |
+
from apis.rest_countries import lookup_country
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def get_country_profiles(countries: list[str]) -> dict[str, Any]:
|
| 12 |
+
profiles: list[dict[str, Any]] = []
|
| 13 |
+
unknown: list[str] = []
|
| 14 |
+
|
| 15 |
+
for raw_code in countries:
|
| 16 |
+
code = (raw_code or "").strip().upper()
|
| 17 |
+
if not code:
|
| 18 |
+
continue
|
| 19 |
+
info = lookup_country(code)
|
| 20 |
+
if not info:
|
| 21 |
+
unknown.append(code)
|
| 22 |
+
continue
|
| 23 |
+
|
| 24 |
+
iso2 = info["cca2"]
|
| 25 |
+
profiles.append(
|
| 26 |
+
{
|
| 27 |
+
"name": info["name"],
|
| 28 |
+
"iso2": iso2,
|
| 29 |
+
"iso3": info["cca3"],
|
| 30 |
+
"region": info.get("region"),
|
| 31 |
+
"subregion": info.get("subregion"),
|
| 32 |
+
"capital": info.get("capital") or [],
|
| 33 |
+
"area_sq_km": info.get("area"),
|
| 34 |
+
"lat": info["lat"],
|
| 35 |
+
"lng": info["lng"],
|
| 36 |
+
"flag": info.get("flag"),
|
| 37 |
+
"official_immigration_domains": hints_for_country(iso2),
|
| 38 |
+
}
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
result: dict[str, Any] = {"countries": profiles}
|
| 42 |
+
if unknown:
|
| 43 |
+
result["unknown_codes"] = unknown
|
| 44 |
+
return result
|
apis/exa.py
CHANGED
|
@@ -8,6 +8,8 @@ from typing import Any
|
|
| 8 |
|
| 9 |
from exa_py import Exa
|
| 10 |
|
|
|
|
|
|
|
| 11 |
HIGHLIGHT_CHAR_LIMIT = 2000
|
| 12 |
SUMMARY_CHAR_LIMIT = 500
|
| 13 |
DEFAULT_NUM_RESULTS = 8
|
|
@@ -36,11 +38,13 @@ def _result_dict(result: Any) -> dict[str, Any]:
|
|
| 36 |
_truncate(highlight, HIGHLIGHT_CHAR_LIMIT) for highlight in highlights
|
| 37 |
]
|
| 38 |
summary = getattr(result, "summary", None)
|
|
|
|
| 39 |
return {
|
| 40 |
"title": result.title or "",
|
| 41 |
"url": result.url or "",
|
| 42 |
"published_date": result.published_date,
|
| 43 |
"score": result.score,
|
|
|
|
| 44 |
"highlights": trimmed_highlights,
|
| 45 |
"summary": _truncate(summary, SUMMARY_CHAR_LIMIT) if summary else None,
|
| 46 |
}
|
|
@@ -88,11 +92,19 @@ def search_immigration(
|
|
| 88 |
if include_domains:
|
| 89 |
kwargs["include_domains"] = include_domains
|
| 90 |
|
|
|
|
|
|
|
| 91 |
response = _client().search(normalized_query, **kwargs)
|
| 92 |
results = [_result_dict(result) for result in (response.results or [])]
|
| 93 |
return {
|
| 94 |
"query": normalized_query,
|
| 95 |
"num_results": len(results),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
"results": results,
|
| 97 |
}
|
| 98 |
except Exception as exc:
|
|
|
|
| 8 |
|
| 9 |
from exa_py import Exa
|
| 10 |
|
| 11 |
+
from apis.official_sources import classify_source, infer_official_domains
|
| 12 |
+
|
| 13 |
HIGHLIGHT_CHAR_LIMIT = 2000
|
| 14 |
SUMMARY_CHAR_LIMIT = 500
|
| 15 |
DEFAULT_NUM_RESULTS = 8
|
|
|
|
| 38 |
_truncate(highlight, HIGHLIGHT_CHAR_LIMIT) for highlight in highlights
|
| 39 |
]
|
| 40 |
summary = getattr(result, "summary", None)
|
| 41 |
+
source_quality = classify_source(result.url or "")
|
| 42 |
return {
|
| 43 |
"title": result.title or "",
|
| 44 |
"url": result.url or "",
|
| 45 |
"published_date": result.published_date,
|
| 46 |
"score": result.score,
|
| 47 |
+
"source_quality": source_quality,
|
| 48 |
"highlights": trimmed_highlights,
|
| 49 |
"summary": _truncate(summary, SUMMARY_CHAR_LIMIT) if summary else None,
|
| 50 |
}
|
|
|
|
| 92 |
if include_domains:
|
| 93 |
kwargs["include_domains"] = include_domains
|
| 94 |
|
| 95 |
+
official_domain_hints = infer_official_domains(normalized_query, country)
|
| 96 |
+
|
| 97 |
response = _client().search(normalized_query, **kwargs)
|
| 98 |
results = [_result_dict(result) for result in (response.results or [])]
|
| 99 |
return {
|
| 100 |
"query": normalized_query,
|
| 101 |
"num_results": len(results),
|
| 102 |
+
"official_domain_hints": official_domain_hints,
|
| 103 |
+
"official_results": sum(
|
| 104 |
+
1
|
| 105 |
+
for result in results
|
| 106 |
+
if result.get("source_quality", {}).get("is_official")
|
| 107 |
+
),
|
| 108 |
"results": results,
|
| 109 |
}
|
| 110 |
except Exception as exc:
|
apis/official_sources.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Helpers for identifying official immigration source domains."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from urllib.parse import urlparse
|
| 6 |
+
|
| 7 |
+
OFFICIAL_DOMAIN_HINTS: dict[str, list[str]] = {
|
| 8 |
+
"AU": ["homeaffairs.gov.au", "immi.homeaffairs.gov.au"],
|
| 9 |
+
"CA": ["canada.ca", "cic.gc.ca"],
|
| 10 |
+
"DE": ["make-it-in-germany.com", "bamf.de", "auswaertiges-amt.de"],
|
| 11 |
+
"DK": ["nyidanmark.dk"],
|
| 12 |
+
"ES": ["inclusion.gob.es", "exteriores.gob.es"],
|
| 13 |
+
"FI": ["migri.fi"],
|
| 14 |
+
"FR": ["france-visas.gouv.fr", "service-public.fr"],
|
| 15 |
+
"GB": ["gov.uk"],
|
| 16 |
+
"IE": ["irishimmigration.ie", "enterprise.gov.ie"],
|
| 17 |
+
"JP": ["isa.go.jp", "mofa.go.jp"],
|
| 18 |
+
"NL": ["ind.nl"],
|
| 19 |
+
"NO": ["udi.no"],
|
| 20 |
+
"NZ": ["immigration.govt.nz"],
|
| 21 |
+
"PT": ["aima.gov.pt", "eportugal.gov.pt", "vistos.mne.gov.pt"],
|
| 22 |
+
"SE": ["migrationsverket.se"],
|
| 23 |
+
"SG": ["ica.gov.sg", "mom.gov.sg"],
|
| 24 |
+
"US": ["uscis.gov", "travel.state.gov"],
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
COUNTRY_ALIASES: dict[str, str] = {
|
| 28 |
+
"australia": "AU",
|
| 29 |
+
"canada": "CA",
|
| 30 |
+
"denmark": "DK",
|
| 31 |
+
"finland": "FI",
|
| 32 |
+
"france": "FR",
|
| 33 |
+
"germany": "DE",
|
| 34 |
+
"ireland": "IE",
|
| 35 |
+
"japan": "JP",
|
| 36 |
+
"netherlands": "NL",
|
| 37 |
+
"new zealand": "NZ",
|
| 38 |
+
"norway": "NO",
|
| 39 |
+
"portugal": "PT",
|
| 40 |
+
"singapore": "SG",
|
| 41 |
+
"spain": "ES",
|
| 42 |
+
"sweden": "SE",
|
| 43 |
+
"uk": "GB",
|
| 44 |
+
"united kingdom": "GB",
|
| 45 |
+
"united states": "US",
|
| 46 |
+
"usa": "US",
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
UNOFFICIAL_CONTEXT_TERMS = (
|
| 50 |
+
"blog",
|
| 51 |
+
"forum",
|
| 52 |
+
"reddit",
|
| 53 |
+
"quora",
|
| 54 |
+
"lawfirm",
|
| 55 |
+
"law-firm",
|
| 56 |
+
"immigrationlaw",
|
| 57 |
+
"relocation",
|
| 58 |
+
"movingto",
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def domain_from_url(url: str) -> str:
|
| 63 |
+
parsed = urlparse(url or "")
|
| 64 |
+
host = parsed.netloc.lower().split("@")[-1].split(":")[0]
|
| 65 |
+
if host.startswith("www."):
|
| 66 |
+
host = host[4:]
|
| 67 |
+
return host
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def is_domain_match(domain: str, official_domain: str) -> bool:
|
| 71 |
+
normalized = official_domain.lower()
|
| 72 |
+
return domain == normalized or domain.endswith(f".{normalized}")
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def hints_for_country(country: str | None) -> list[str]:
|
| 76 |
+
if not country:
|
| 77 |
+
return []
|
| 78 |
+
normalized = country.strip().upper()
|
| 79 |
+
return OFFICIAL_DOMAIN_HINTS.get(normalized, [])
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def infer_country_codes(text: str, country: str | None = None) -> list[str]:
|
| 83 |
+
codes: list[str] = []
|
| 84 |
+
if country:
|
| 85 |
+
normalized = country.strip().upper()
|
| 86 |
+
if normalized in OFFICIAL_DOMAIN_HINTS:
|
| 87 |
+
codes.append(normalized)
|
| 88 |
+
|
| 89 |
+
lowered = (text or "").lower()
|
| 90 |
+
for alias, code in COUNTRY_ALIASES.items():
|
| 91 |
+
if alias in lowered and code not in codes:
|
| 92 |
+
codes.append(code)
|
| 93 |
+
return codes
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def infer_official_domains(
|
| 97 |
+
text: str,
|
| 98 |
+
country: str | None = None,
|
| 99 |
+
*,
|
| 100 |
+
max_domains: int = 8,
|
| 101 |
+
) -> list[str]:
|
| 102 |
+
domains: list[str] = []
|
| 103 |
+
for code in infer_country_codes(text, country):
|
| 104 |
+
for domain in OFFICIAL_DOMAIN_HINTS.get(code, []):
|
| 105 |
+
if domain not in domains:
|
| 106 |
+
domains.append(domain)
|
| 107 |
+
if len(domains) >= max_domains:
|
| 108 |
+
return domains
|
| 109 |
+
return domains
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def classify_source(url: str) -> dict[str, object]:
|
| 113 |
+
domain = domain_from_url(url)
|
| 114 |
+
known_official = [
|
| 115 |
+
official_domain
|
| 116 |
+
for domains in OFFICIAL_DOMAIN_HINTS.values()
|
| 117 |
+
for official_domain in domains
|
| 118 |
+
if is_domain_match(domain, official_domain)
|
| 119 |
+
]
|
| 120 |
+
|
| 121 |
+
if known_official:
|
| 122 |
+
return {
|
| 123 |
+
"domain": domain,
|
| 124 |
+
"type": "official_government",
|
| 125 |
+
"is_official": True,
|
| 126 |
+
"reason": f"Matches official domain hint {known_official[0]}",
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
if domain.endswith(".gov") or ".gov." in domain or domain.endswith(".gouv.fr"):
|
| 130 |
+
return {
|
| 131 |
+
"domain": domain,
|
| 132 |
+
"type": "government",
|
| 133 |
+
"is_official": True,
|
| 134 |
+
"reason": "Government domain pattern",
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
if "embassy" in domain or "consulate" in domain:
|
| 138 |
+
return {
|
| 139 |
+
"domain": domain,
|
| 140 |
+
"type": "embassy_or_consulate",
|
| 141 |
+
"is_official": True,
|
| 142 |
+
"reason": "Embassy or consulate domain pattern",
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
if any(term in domain for term in UNOFFICIAL_CONTEXT_TERMS):
|
| 146 |
+
return {
|
| 147 |
+
"domain": domain,
|
| 148 |
+
"type": "unofficial_context",
|
| 149 |
+
"is_official": False,
|
| 150 |
+
"reason": "Likely blog, forum, legal-service, or relocation context",
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
return {
|
| 154 |
+
"domain": domain,
|
| 155 |
+
"type": "unknown",
|
| 156 |
+
"is_official": False,
|
| 157 |
+
"reason": "No official-domain signal detected",
|
| 158 |
+
}
|
apis/rest_countries.py
CHANGED
|
@@ -41,6 +41,9 @@ def _country_index() -> dict[str, dict[str, Any]]:
|
|
| 41 |
"subregion": country.get("subregion"),
|
| 42 |
"capital": country.get("capital") or [],
|
| 43 |
"area": country.get("area"),
|
|
|
|
|
|
|
|
|
|
| 44 |
}
|
| 45 |
|
| 46 |
if entry["cca2"]:
|
|
|
|
| 41 |
"subregion": country.get("subregion"),
|
| 42 |
"capital": country.get("capital") or [],
|
| 43 |
"area": country.get("area"),
|
| 44 |
+
"flag": (country.get("flags") or {}).get("png")
|
| 45 |
+
or (country.get("flags") or {}).get("svg"),
|
| 46 |
+
"flag_alt": (country.get("flags") or {}).get("alt"),
|
| 47 |
}
|
| 48 |
|
| 49 |
if entry["cca2"]:
|
ui/inference/tool_schemas/__init__.py
CHANGED
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
| 3 |
from typing import Any
|
| 4 |
|
| 5 |
from .crawl_web_site import SCHEMA as crawl_web_site
|
|
|
|
| 6 |
from .scrape_web_page import SCHEMA as scrape_web_page
|
| 7 |
from .search_immigration_info import SCHEMA as search_immigration_info
|
| 8 |
from .think import SCHEMA as think
|
|
@@ -10,6 +11,7 @@ from .update_globe import SCHEMA as update_globe
|
|
| 10 |
|
| 11 |
TOOL_SCHEMAS: list[dict[str, Any]] = [
|
| 12 |
think,
|
|
|
|
| 13 |
search_immigration_info,
|
| 14 |
scrape_web_page,
|
| 15 |
crawl_web_site,
|
|
|
|
| 3 |
from typing import Any
|
| 4 |
|
| 5 |
from .crawl_web_site import SCHEMA as crawl_web_site
|
| 6 |
+
from .get_country_profile import SCHEMA as get_country_profile
|
| 7 |
from .scrape_web_page import SCHEMA as scrape_web_page
|
| 8 |
from .search_immigration_info import SCHEMA as search_immigration_info
|
| 9 |
from .think import SCHEMA as think
|
|
|
|
| 11 |
|
| 12 |
TOOL_SCHEMAS: list[dict[str, Any]] = [
|
| 13 |
think,
|
| 14 |
+
get_country_profile,
|
| 15 |
search_immigration_info,
|
| 16 |
scrape_web_page,
|
| 17 |
crawl_web_site,
|
ui/inference/tool_schemas/get_country_profile.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
SCHEMA: dict[str, Any] = {
|
| 6 |
+
"type": "function",
|
| 7 |
+
"function": {
|
| 8 |
+
"name": "get_country_profile",
|
| 9 |
+
"description": (
|
| 10 |
+
"Fetch basic country metadata and official immigration domain hints. "
|
| 11 |
+
"Use this before comparing or marking destination countries on the globe."
|
| 12 |
+
),
|
| 13 |
+
"parameters": {
|
| 14 |
+
"type": "object",
|
| 15 |
+
"properties": {
|
| 16 |
+
"countries": {
|
| 17 |
+
"type": "array",
|
| 18 |
+
"items": {"type": "string"},
|
| 19 |
+
"description": "ISO-2 or ISO-3 country codes, e.g. ['CA', 'DE', 'AUS']",
|
| 20 |
+
},
|
| 21 |
+
},
|
| 22 |
+
"required": ["countries"],
|
| 23 |
+
},
|
| 24 |
+
},
|
| 25 |
+
}
|
ui/inference/tool_schemas/search_immigration_info.py
CHANGED
|
@@ -10,8 +10,9 @@ SCHEMA: dict[str, Any] = {
|
|
| 10 |
"Search the web for immigration, visa, and relocation information. "
|
| 11 |
"Use this as the primary discovery tool when you need to find "
|
| 12 |
"official government pages, embassy sites, or current policy details. "
|
| 13 |
-
"Returns titles, URLs,
|
| 14 |
-
"
|
|
|
|
| 15 |
),
|
| 16 |
"parameters": {
|
| 17 |
"type": "object",
|
|
|
|
| 10 |
"Search the web for immigration, visa, and relocation information. "
|
| 11 |
"Use this as the primary discovery tool when you need to find "
|
| 12 |
"official government pages, embassy sites, or current policy details. "
|
| 13 |
+
"Returns titles, URLs, relevant excerpts, source-quality labels, and "
|
| 14 |
+
"official domain hints. Follow up with scrape_web_page on the best "
|
| 15 |
+
"official URLs for full page content."
|
| 16 |
),
|
| 17 |
"parameters": {
|
| 18 |
"type": "object",
|