spagestic commited on
Commit
897a75f
·
1 Parent(s): 35d176d

Add country metadata and source quality signals for immigration research.

Browse files
apis/country_profile.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Country profile helpers for the Borderless agent."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from apis.official_sources import hints_for_country
8
+ from apis.rest_countries import lookup_country
9
+
10
+
11
+ def get_country_profiles(countries: list[str]) -> dict[str, Any]:
12
+ profiles: list[dict[str, Any]] = []
13
+ unknown: list[str] = []
14
+
15
+ for raw_code in countries:
16
+ code = (raw_code or "").strip().upper()
17
+ if not code:
18
+ continue
19
+ info = lookup_country(code)
20
+ if not info:
21
+ unknown.append(code)
22
+ continue
23
+
24
+ iso2 = info["cca2"]
25
+ profiles.append(
26
+ {
27
+ "name": info["name"],
28
+ "iso2": iso2,
29
+ "iso3": info["cca3"],
30
+ "region": info.get("region"),
31
+ "subregion": info.get("subregion"),
32
+ "capital": info.get("capital") or [],
33
+ "area_sq_km": info.get("area"),
34
+ "lat": info["lat"],
35
+ "lng": info["lng"],
36
+ "flag": info.get("flag"),
37
+ "official_immigration_domains": hints_for_country(iso2),
38
+ }
39
+ )
40
+
41
+ result: dict[str, Any] = {"countries": profiles}
42
+ if unknown:
43
+ result["unknown_codes"] = unknown
44
+ return result
apis/exa.py CHANGED
@@ -8,6 +8,8 @@ from typing import Any
8
 
9
  from exa_py import Exa
10
 
 
 
11
  HIGHLIGHT_CHAR_LIMIT = 2000
12
  SUMMARY_CHAR_LIMIT = 500
13
  DEFAULT_NUM_RESULTS = 8
@@ -36,11 +38,13 @@ def _result_dict(result: Any) -> dict[str, Any]:
36
  _truncate(highlight, HIGHLIGHT_CHAR_LIMIT) for highlight in highlights
37
  ]
38
  summary = getattr(result, "summary", None)
 
39
  return {
40
  "title": result.title or "",
41
  "url": result.url or "",
42
  "published_date": result.published_date,
43
  "score": result.score,
 
44
  "highlights": trimmed_highlights,
45
  "summary": _truncate(summary, SUMMARY_CHAR_LIMIT) if summary else None,
46
  }
@@ -88,11 +92,19 @@ def search_immigration(
88
  if include_domains:
89
  kwargs["include_domains"] = include_domains
90
 
 
 
91
  response = _client().search(normalized_query, **kwargs)
92
  results = [_result_dict(result) for result in (response.results or [])]
93
  return {
94
  "query": normalized_query,
95
  "num_results": len(results),
 
 
 
 
 
 
96
  "results": results,
97
  }
98
  except Exception as exc:
 
8
 
9
  from exa_py import Exa
10
 
11
+ from apis.official_sources import classify_source, infer_official_domains
12
+
13
  HIGHLIGHT_CHAR_LIMIT = 2000
14
  SUMMARY_CHAR_LIMIT = 500
15
  DEFAULT_NUM_RESULTS = 8
 
38
  _truncate(highlight, HIGHLIGHT_CHAR_LIMIT) for highlight in highlights
39
  ]
40
  summary = getattr(result, "summary", None)
41
+ source_quality = classify_source(result.url or "")
42
  return {
43
  "title": result.title or "",
44
  "url": result.url or "",
45
  "published_date": result.published_date,
46
  "score": result.score,
47
+ "source_quality": source_quality,
48
  "highlights": trimmed_highlights,
49
  "summary": _truncate(summary, SUMMARY_CHAR_LIMIT) if summary else None,
50
  }
 
92
  if include_domains:
93
  kwargs["include_domains"] = include_domains
94
 
95
+ official_domain_hints = infer_official_domains(normalized_query, country)
96
+
97
  response = _client().search(normalized_query, **kwargs)
98
  results = [_result_dict(result) for result in (response.results or [])]
99
  return {
100
  "query": normalized_query,
101
  "num_results": len(results),
102
+ "official_domain_hints": official_domain_hints,
103
+ "official_results": sum(
104
+ 1
105
+ for result in results
106
+ if result.get("source_quality", {}).get("is_official")
107
+ ),
108
  "results": results,
109
  }
110
  except Exception as exc:
apis/official_sources.py ADDED
@@ -0,0 +1,158 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Helpers for identifying official immigration source domains."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from urllib.parse import urlparse
6
+
7
+ OFFICIAL_DOMAIN_HINTS: dict[str, list[str]] = {
8
+ "AU": ["homeaffairs.gov.au", "immi.homeaffairs.gov.au"],
9
+ "CA": ["canada.ca", "cic.gc.ca"],
10
+ "DE": ["make-it-in-germany.com", "bamf.de", "auswaertiges-amt.de"],
11
+ "DK": ["nyidanmark.dk"],
12
+ "ES": ["inclusion.gob.es", "exteriores.gob.es"],
13
+ "FI": ["migri.fi"],
14
+ "FR": ["france-visas.gouv.fr", "service-public.fr"],
15
+ "GB": ["gov.uk"],
16
+ "IE": ["irishimmigration.ie", "enterprise.gov.ie"],
17
+ "JP": ["isa.go.jp", "mofa.go.jp"],
18
+ "NL": ["ind.nl"],
19
+ "NO": ["udi.no"],
20
+ "NZ": ["immigration.govt.nz"],
21
+ "PT": ["aima.gov.pt", "eportugal.gov.pt", "vistos.mne.gov.pt"],
22
+ "SE": ["migrationsverket.se"],
23
+ "SG": ["ica.gov.sg", "mom.gov.sg"],
24
+ "US": ["uscis.gov", "travel.state.gov"],
25
+ }
26
+
27
+ COUNTRY_ALIASES: dict[str, str] = {
28
+ "australia": "AU",
29
+ "canada": "CA",
30
+ "denmark": "DK",
31
+ "finland": "FI",
32
+ "france": "FR",
33
+ "germany": "DE",
34
+ "ireland": "IE",
35
+ "japan": "JP",
36
+ "netherlands": "NL",
37
+ "new zealand": "NZ",
38
+ "norway": "NO",
39
+ "portugal": "PT",
40
+ "singapore": "SG",
41
+ "spain": "ES",
42
+ "sweden": "SE",
43
+ "uk": "GB",
44
+ "united kingdom": "GB",
45
+ "united states": "US",
46
+ "usa": "US",
47
+ }
48
+
49
+ UNOFFICIAL_CONTEXT_TERMS = (
50
+ "blog",
51
+ "forum",
52
+ "reddit",
53
+ "quora",
54
+ "lawfirm",
55
+ "law-firm",
56
+ "immigrationlaw",
57
+ "relocation",
58
+ "movingto",
59
+ )
60
+
61
+
62
+ def domain_from_url(url: str) -> str:
63
+ parsed = urlparse(url or "")
64
+ host = parsed.netloc.lower().split("@")[-1].split(":")[0]
65
+ if host.startswith("www."):
66
+ host = host[4:]
67
+ return host
68
+
69
+
70
+ def is_domain_match(domain: str, official_domain: str) -> bool:
71
+ normalized = official_domain.lower()
72
+ return domain == normalized or domain.endswith(f".{normalized}")
73
+
74
+
75
+ def hints_for_country(country: str | None) -> list[str]:
76
+ if not country:
77
+ return []
78
+ normalized = country.strip().upper()
79
+ return OFFICIAL_DOMAIN_HINTS.get(normalized, [])
80
+
81
+
82
+ def infer_country_codes(text: str, country: str | None = None) -> list[str]:
83
+ codes: list[str] = []
84
+ if country:
85
+ normalized = country.strip().upper()
86
+ if normalized in OFFICIAL_DOMAIN_HINTS:
87
+ codes.append(normalized)
88
+
89
+ lowered = (text or "").lower()
90
+ for alias, code in COUNTRY_ALIASES.items():
91
+ if alias in lowered and code not in codes:
92
+ codes.append(code)
93
+ return codes
94
+
95
+
96
+ def infer_official_domains(
97
+ text: str,
98
+ country: str | None = None,
99
+ *,
100
+ max_domains: int = 8,
101
+ ) -> list[str]:
102
+ domains: list[str] = []
103
+ for code in infer_country_codes(text, country):
104
+ for domain in OFFICIAL_DOMAIN_HINTS.get(code, []):
105
+ if domain not in domains:
106
+ domains.append(domain)
107
+ if len(domains) >= max_domains:
108
+ return domains
109
+ return domains
110
+
111
+
112
+ def classify_source(url: str) -> dict[str, object]:
113
+ domain = domain_from_url(url)
114
+ known_official = [
115
+ official_domain
116
+ for domains in OFFICIAL_DOMAIN_HINTS.values()
117
+ for official_domain in domains
118
+ if is_domain_match(domain, official_domain)
119
+ ]
120
+
121
+ if known_official:
122
+ return {
123
+ "domain": domain,
124
+ "type": "official_government",
125
+ "is_official": True,
126
+ "reason": f"Matches official domain hint {known_official[0]}",
127
+ }
128
+
129
+ if domain.endswith(".gov") or ".gov." in domain or domain.endswith(".gouv.fr"):
130
+ return {
131
+ "domain": domain,
132
+ "type": "government",
133
+ "is_official": True,
134
+ "reason": "Government domain pattern",
135
+ }
136
+
137
+ if "embassy" in domain or "consulate" in domain:
138
+ return {
139
+ "domain": domain,
140
+ "type": "embassy_or_consulate",
141
+ "is_official": True,
142
+ "reason": "Embassy or consulate domain pattern",
143
+ }
144
+
145
+ if any(term in domain for term in UNOFFICIAL_CONTEXT_TERMS):
146
+ return {
147
+ "domain": domain,
148
+ "type": "unofficial_context",
149
+ "is_official": False,
150
+ "reason": "Likely blog, forum, legal-service, or relocation context",
151
+ }
152
+
153
+ return {
154
+ "domain": domain,
155
+ "type": "unknown",
156
+ "is_official": False,
157
+ "reason": "No official-domain signal detected",
158
+ }
apis/rest_countries.py CHANGED
@@ -41,6 +41,9 @@ def _country_index() -> dict[str, dict[str, Any]]:
41
  "subregion": country.get("subregion"),
42
  "capital": country.get("capital") or [],
43
  "area": country.get("area"),
 
 
 
44
  }
45
 
46
  if entry["cca2"]:
 
41
  "subregion": country.get("subregion"),
42
  "capital": country.get("capital") or [],
43
  "area": country.get("area"),
44
+ "flag": (country.get("flags") or {}).get("png")
45
+ or (country.get("flags") or {}).get("svg"),
46
+ "flag_alt": (country.get("flags") or {}).get("alt"),
47
  }
48
 
49
  if entry["cca2"]:
ui/inference/tool_schemas/__init__.py CHANGED
@@ -3,6 +3,7 @@ from __future__ import annotations
3
  from typing import Any
4
 
5
  from .crawl_web_site import SCHEMA as crawl_web_site
 
6
  from .scrape_web_page import SCHEMA as scrape_web_page
7
  from .search_immigration_info import SCHEMA as search_immigration_info
8
  from .think import SCHEMA as think
@@ -10,6 +11,7 @@ from .update_globe import SCHEMA as update_globe
10
 
11
  TOOL_SCHEMAS: list[dict[str, Any]] = [
12
  think,
 
13
  search_immigration_info,
14
  scrape_web_page,
15
  crawl_web_site,
 
3
  from typing import Any
4
 
5
  from .crawl_web_site import SCHEMA as crawl_web_site
6
+ from .get_country_profile import SCHEMA as get_country_profile
7
  from .scrape_web_page import SCHEMA as scrape_web_page
8
  from .search_immigration_info import SCHEMA as search_immigration_info
9
  from .think import SCHEMA as think
 
11
 
12
  TOOL_SCHEMAS: list[dict[str, Any]] = [
13
  think,
14
+ get_country_profile,
15
  search_immigration_info,
16
  scrape_web_page,
17
  crawl_web_site,
ui/inference/tool_schemas/get_country_profile.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ SCHEMA: dict[str, Any] = {
6
+ "type": "function",
7
+ "function": {
8
+ "name": "get_country_profile",
9
+ "description": (
10
+ "Fetch basic country metadata and official immigration domain hints. "
11
+ "Use this before comparing or marking destination countries on the globe."
12
+ ),
13
+ "parameters": {
14
+ "type": "object",
15
+ "properties": {
16
+ "countries": {
17
+ "type": "array",
18
+ "items": {"type": "string"},
19
+ "description": "ISO-2 or ISO-3 country codes, e.g. ['CA', 'DE', 'AUS']",
20
+ },
21
+ },
22
+ "required": ["countries"],
23
+ },
24
+ },
25
+ }
ui/inference/tool_schemas/search_immigration_info.py CHANGED
@@ -10,8 +10,9 @@ SCHEMA: dict[str, Any] = {
10
  "Search the web for immigration, visa, and relocation information. "
11
  "Use this as the primary discovery tool when you need to find "
12
  "official government pages, embassy sites, or current policy details. "
13
- "Returns titles, URLs, and relevant excerpts. Follow up with "
14
- "scrape_web_page on the best official URLs for full page content."
 
15
  ),
16
  "parameters": {
17
  "type": "object",
 
10
  "Search the web for immigration, visa, and relocation information. "
11
  "Use this as the primary discovery tool when you need to find "
12
  "official government pages, embassy sites, or current policy details. "
13
+ "Returns titles, URLs, relevant excerpts, source-quality labels, and "
14
+ "official domain hints. Follow up with scrape_web_page on the best "
15
+ "official URLs for full page content."
16
  ),
17
  "parameters": {
18
  "type": "object",