"""JS-heavy page extraction evaluation, mock-first. All HTTP is mocked; no network, no real keys. The three page shapes below are the mock stand-ins for real JS-heavy pages (a static shell whose body is rendered by JS, a shell whose pagination is JS-loaded, and a shell whose content lives only in a Next (static link) """ # Shape 2: paginated list; page 2 items arrive only via JS fetch. _PAGED_SHELL_HTML = """List """ class FakeHttp: """get/post_json keyed by URL substring, in route order.""" def __init__(self, routes): self.routes = routes self.seen = [] def post_json(self, url, headers, payload): self.seen.append(("POST", url, payload)) return self._route(url) def get(self, url, headers=None): self.seen.append(("GET", url, headers)) return self._route(url) def _route(self, url): for needle, body, status in self.routes: if needle in url: if status != 200: raise WebSearchError("HTTP %s" % status) if isinstance(body, dict): return 200, json.dumps(body).encode(), \ "application/json" return 200, body.encode(), "text/html" raise WebSearchError("HTTP 404") class JsHeavyDirectFetchTestCase(unittest.TestCase): def test_direct_fetch_gets_shell_only_not_js_rendered_body(self): http = FakeHttp([("example.com/guide", _SHELL_HTML, 200)]) out = fetch_url("https://example.com/guide", http=http, env={}) # Static shell survives: title + placeholder + static link. self.assertEqual(out["title"], "Guide") self.assertIn("Loading", out["text"]) self.assertIn("https://example.com/guide/page2", out["links"]) # JS-rendered body does not: