Add page extraction tool

tests/test_config_web_search.py

@@ -29,6 +29,10 @@ def _raw_ai_config() -> dict:
             "count": 99,
             "safesearch": "strict",
         },
+        "page_extract": {
+            "max_chars": 5000,
+            "render_dynamic": False,
+        },
     }
 
 
@@ -41,6 +45,8 @@ class ConfigWebSearchTests(unittest.TestCase):
         self.assertEqual(config.web_search.api_key, "config-key")
         self.assertEqual(config.web_search.count, 20)
         self.assertEqual(config.web_search.safesearch, "strict")
+        self.assertEqual(config.page_extract.max_chars, 5000)
+        self.assertFalse(config.page_extract.render_dynamic)
 
     def test_web_search_env_overrides(self) -> None:
         env = {
@@ -50,6 +56,8 @@ class ConfigWebSearchTests(unittest.TestCase):
             "BRAVE_SEARCH_LANG": "pl",
             "BRAVE_SEARCH_UI_LANG": "pl-PL",
             "BRAVE_SEARCH_SAFESEARCH": "off",
+            "PAGE_EXTRACT_MAX_CHARS": "9000",
+            "PAGE_EXTRACT_RENDER_DYNAMIC": "true",
         }
         with patch.dict("os.environ", env, clear=True):
             config = _parse_ai_config(_raw_ai_config(), root=Path.cwd())
@@ -61,6 +69,8 @@ class ConfigWebSearchTests(unittest.TestCase):
         self.assertEqual(config.web_search.search_lang, "pl")
         self.assertEqual(config.web_search.ui_lang, "pl-PL")
         self.assertEqual(config.web_search.safesearch, "off")
+        self.assertEqual(config.page_extract.max_chars, 9000)
+        self.assertTrue(config.page_extract.render_dynamic)
 
 
 if __name__ == "__main__":

tests/test_page_extract.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import unittest
+
+from app.ai.page_extract import extract_readable_text, normalize_url, validate_public_url
+
+
+class PageExtractTests(unittest.TestCase):
+    def test_extract_readable_text_skips_scripts_and_styles(self) -> None:
+        title, text = extract_readable_text(
+            """
+            <html>
+              <head>
+                <title>Example Page</title>
+                <style>.hidden { display: none; }</style>
+              </head>
+              <body>
+                <h1>Hello</h1>
+                <script>alert('nope')</script>
+                <p>Readable text.</p>
+              </body>
+            </html>
+            """
+        )
+
+        self.assertEqual(title, "Example Page")
+        self.assertIn("Hello", text)
+        self.assertIn("Readable text.", text)
+        self.assertNotIn("alert", text)
+        self.assertNotIn("display", text)
+
+    def test_normalize_url_rejects_non_web_schemes(self) -> None:
+        with self.assertRaisesRegex(ValueError, "http and https"):
+            normalize_url("file:///etc/passwd")
+
+    def test_validate_public_url_blocks_localhost(self) -> None:
+        with self.assertRaisesRegex(ValueError, "blocked"):
+            validate_public_url("http://127.0.0.1:8000", block_private_hosts=True)
+
+    def test_validate_public_url_can_allow_private_hosts(self) -> None:
+        validate_public_url("http://127.0.0.1:8000", block_private_hosts=False)
+
+
+if __name__ == "__main__":
+    unittest.main()