codex-app-server/cloud/tests/test_js_heavy_extract.py

"""JS-heavy page extraction evaluation, mock-first.

All HTTP is mocked; no network, no real keys. The three page shapes
below are the mock stand-ins for real JS-heavy pages (a static shell
whose body is rendered by JS, a shell whose pagination is JS-loaded,
and a shell whose content lives only in a <script> blob):

- direct fetch (fetch_url) parses static HTML only: <script> and
  <style> contents are stripped, so anything JS would have rendered
  is absent from the result -- this is by construction, not a bug;
- a provider extract stage (fetch_via_provider, mocked here) can
  return the rendered text when the provider actually renders, and
  an empty extraction is a failure, never a fabricated page.

Live对照: only the keyless SearXNG route is ever used live, and
only when a test environment actually provides SEARXNG_URL; this
file never calls a paid backend live. Set
WEBSEARCH_EVAL_LIVE_SEARXNG=1 with SEARXNG_URL in the environment
to enable the live smoke test, otherwise it is skipped.
"""

import json
import os
import sys
import unittest
from pathlib import Path

try:
    from cloud.websearch import (
        WebSearchError, fetch_best, fetch_url, fetch_via_provider)
except ImportError:
    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
    from cloud.websearch import (
        WebSearchError, fetch_best, fetch_url, fetch_via_provider)

_FAKE_KEYS = {
    "TAVILY_API_KEY": "tvly-SECRET-333",
    "EXA_API_KEY": "exa-SECRET-222",
}

# Shape 1: shell HTML only; the article body is JS-rendered.
_SHELL_HTML = """<html><head><title>Guide</title></head><body>
<div id="app">Loading…</div>
<script src="/app.js"></script>
<script>window.__DATA__ = {"body": "RENDERED-BODY-PARAGRAPH"};</script>
<a href="https://example.com/guide/page2">Next (static link)</a>
</body></html>"""

# Shape 2: paginated list; page 2 items arrive only via JS fetch.
_PAGED_SHELL_HTML = """<html><head><title>List</title></head><body>
<ul id="items"><li>ITEM-PAGE-1</li></ul>
<button id="more">Load more</button>
<script>fetch("/api/items?page=2").then(r => r.json());</script>
</body></html>"""


class FakeHttp:
    """get/post_json keyed by URL substring, in route order."""

    def __init__(self, routes):
        self.routes = routes
        self.seen = []

    def post_json(self, url, headers, payload):
        self.seen.append(("POST", url, payload))
        return self._route(url)

    def get(self, url, headers=None):
        self.seen.append(("GET", url, headers))
        return self._route(url)

    def _route(self, url):
        for needle, body, status in self.routes:
            if needle in url:
                if status != 200:
                    raise WebSearchError("HTTP %s" % status)
                if isinstance(body, dict):
                    return 200, json.dumps(body).encode(), \
                        "application/json"
                return 200, body.encode(), "text/html"
        raise WebSearchError("HTTP 404")


class JsHeavyDirectFetchTestCase(unittest.TestCase):
    def test_direct_fetch_gets_shell_only_not_js_rendered_body(self):
        http = FakeHttp([("example.com/guide", _SHELL_HTML, 200)])
        out = fetch_url("https://example.com/guide", http=http, env={})
        # Static shell survives: title + placeholder + static link.
        self.assertEqual(out["title"], "Guide")
        self.assertIn("Loading", out["text"])
        self.assertIn("https://example.com/guide/page2", out["links"])
        # JS-rendered body does not: <script> contents are stripped,
        # so neither the rendered paragraph nor the data blob leaks
        # in as text.
        self.assertNotIn("RENDERED-BODY-PARAGRAPH", out["text"])
        self.assertNotIn("window.__DATA__", out["text"])

    def test_direct_fetch_gets_page1_only_not_js_paginated_items(self):
        http = FakeHttp([("example.com/list", _PAGED_SHELL_HTML, 200)])
        out = fetch_url("https://example.com/list", http=http, env={})
        self.assertIn("ITEM-PAGE-1", out["text"])
        self.assertNotIn("ITEM-PAGE-2", out["text"])
        self.assertNotIn("/api/items?page=2", out["text"])


class JsHeavyProviderExtractTestCase(unittest.TestCase):
    def test_mock_provider_extract_can_supply_rendered_text(self):
        # Mock stand-in for "provider renders JS": Tavily extract
        # returns the full rendered article text.
        http = FakeHttp([
            ("api.tavily.com/extract",
             {"results": [{"raw_content": "RENDERED-BODY-PARAGRAPH "
                                          "plus ITEM-PAGE-2"}]}, 200),
        ])
        out = fetch_via_provider("https://example.com/guide",
                                 env=dict(_FAKE_KEYS), http=http)
        self.assertEqual(out["source"], "tavily-extract")
        self.assertIn("RENDERED-BODY-PARAGRAPH", out["text"])

    def test_mock_provider_extract_empty_is_failure_not_fabrication(self):
        http = FakeHttp([
            ("api.tavily.com/extract", {"results": []}, 200),
            ("api.exa.ai/contents", {"results": []}, 200),
        ])
        with self.assertRaises(WebSearchError):
            fetch_via_provider("https://example.com/guide",
                               env=dict(_FAKE_KEYS), http=http)

    def test_fetch_best_falls_back_to_shell_when_extract_fails(self):
        http = FakeHttp([
            ("api.tavily.com/extract", {"results": []}, 200),
            ("api.exa.ai/contents", {"results": []}, 200),
            ("example.com/guide", _SHELL_HTML, 200),
        ])
        out = fetch_best("https://example.com/guide",
                         env=dict(_FAKE_KEYS), http=http)
        self.assertEqual(out["source"], "direct-fetch")
        self.assertNotIn("RENDERED-BODY-PARAGRAPH", out["text"])


@unittest.skipUnless(
    os.environ.get("WEBSEARCH_EVAL_LIVE_SEARXNG") == "1"
    and os.environ.get("SEARXNG_URL"),
    "live SearXNG对照 only when the test env provides SEARXNG_URL")
class LiveSearxngSmokeTestCase(unittest.TestCase):
    def test_live_searxng_chain_answers_without_paid_keys(self):
        from cloud.websearch import search
        out = search("python pathlib documentation", provider="searxng",
                     scope="general",
                     env={"SEARXNG_URL": os.environ["SEARXNG_URL"]})
        self.assertTrue(out["results"])


if __name__ == "__main__":
    unittest.main()