Add page extraction tool

app/config.py

@@ -78,6 +78,16 @@ class AIWebSearchConfig:
         return self.enabled and bool(self.api_key)
 
 
+@dataclass(slots=True)
+class AIPageExtractConfig:
+    enabled: bool = True
+    timeout: float = 15.0
+    max_chars: int = 12000
+    render_dynamic: bool = True
+    dynamic_wait_ms: int = 1500
+    block_private_hosts: bool = True
+
+
 @dataclass(slots=True)
 class AIConfig:
     default_model: str
@@ -88,6 +98,7 @@ class AIConfig:
     workspace_dir: Path = field(default_factory=lambda: Path("workspace"))
     tool_max_rounds: int = 10
     web_search: AIWebSearchConfig = field(default_factory=AIWebSearchConfig)
+    page_extract: AIPageExtractConfig = field(default_factory=AIPageExtractConfig)
 
 
 @dataclass(slots=True)
@@ -231,6 +242,52 @@ def _parse_web_search_config(raw_ai: dict[str, Any]) -> AIWebSearchConfig:
     )
 
 
+def _parse_page_extract_config(raw_ai: dict[str, Any]) -> AIPageExtractConfig:
+    raw = _parse_mapping(raw_ai.get("page_extract"), field_name="ai.page_extract")
+
+    enabled = _parse_bool(raw.get("enabled", True), field_name="ai.page_extract.enabled")
+    timeout = _parse_float(raw.get("timeout", 15.0), field_name="ai.page_extract.timeout")
+    if timeout <= 0:
+        raise ValueError("ai.page_extract.timeout must be > 0")
+
+    max_chars = _parse_int(raw.get("max_chars", 12000), field_name="ai.page_extract.max_chars")
+    if max_chars < 1000:
+        raise ValueError("ai.page_extract.max_chars must be >= 1000")
+
+    render_dynamic = _parse_bool(
+        raw.get("render_dynamic", True),
+        field_name="ai.page_extract.render_dynamic",
+    )
+    dynamic_wait_ms = _parse_int(
+        raw.get("dynamic_wait_ms", 1500),
+        field_name="ai.page_extract.dynamic_wait_ms",
+    )
+    dynamic_wait_ms = min(max(dynamic_wait_ms, 0), 10000)
+    block_private_hosts = _parse_bool(
+        raw.get("block_private_hosts", True),
+        field_name="ai.page_extract.block_private_hosts",
+    )
+
+    if os.environ.get("PAGE_EXTRACT_MAX_CHARS") is not None:
+        max_chars = _parse_int(os.environ["PAGE_EXTRACT_MAX_CHARS"], field_name="PAGE_EXTRACT_MAX_CHARS")
+        if max_chars < 1000:
+            raise ValueError("PAGE_EXTRACT_MAX_CHARS must be >= 1000")
+    if os.environ.get("PAGE_EXTRACT_RENDER_DYNAMIC") is not None:
+        render_dynamic = _parse_bool(
+            os.environ["PAGE_EXTRACT_RENDER_DYNAMIC"],
+            field_name="PAGE_EXTRACT_RENDER_DYNAMIC",
+        )
+
+    return AIPageExtractConfig(
+        enabled=enabled,
+        timeout=timeout,
+        max_chars=max_chars,
+        render_dynamic=render_dynamic,
+        dynamic_wait_ms=dynamic_wait_ms,
+        block_private_hosts=block_private_hosts,
+    )
+
+
 def _parse_accounts(raw_accounts: Any, *, default_api_id: int | None, default_api_hash: str | None, default_session_dir: Path) -> list[AccountConfig]:
     if raw_accounts is None:
         return []
@@ -384,6 +441,7 @@ def _parse_ai_config(raw_ai: Any, *, root: Path) -> AIConfig | None:
         raise ValueError("ai.tool_max_rounds must be >= 1")
 
     web_search = _parse_web_search_config(raw_ai)
+    page_extract = _parse_page_extract_config(raw_ai)
 
     return AIConfig(
         default_model=default_model,
@@ -394,6 +452,7 @@ def _parse_ai_config(raw_ai: Any, *, root: Path) -> AIConfig | None:
         workspace_dir=workspace_dir,
         tool_max_rounds=tool_max_rounds,
         web_search=web_search,
+        page_extract=page_extract,
     )