From 54b74721af09de8cdc48fbc4b06a722fc128879c Mon Sep 17 00:00:00 2001 From: Patrice Bechard Date: Wed, 27 Aug 2025 18:27:30 -0400 Subject: [PATCH 1/8] add ui portion of hint labeling (missing backend) --- browsergym/core/src/browsergym/core/env.py | 8 + .../core/src/browsergym/core/hint_labeling.py | 28 ++ .../hint_labeling_files/hint_labeling_ui.html | 313 ++++++++++++++++++ browsergym/core/src/browsergym/utils/obs.py | 7 + .../src/browsergym/experiments/loop.py | 7 +- 5 files changed, 359 insertions(+), 4 deletions(-) create mode 100644 browsergym/core/src/browsergym/core/hint_labeling.py create mode 100644 browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html diff --git a/browsergym/core/src/browsergym/core/env.py b/browsergym/core/src/browsergym/core/env.py index 12c5e8bcb..2f918481e 100644 --- a/browsergym/core/src/browsergym/core/env.py +++ b/browsergym/core/src/browsergym/core/env.py @@ -14,6 +14,7 @@ from .action.base import execute_python_code from .action.highlevel import HighLevelActionSet from .chat import Chat +from .hint_labeling import HintLabeling from .constants import BROWSERGYM_ID_ATTRIBUTE, EXTRACT_OBS_MAX_TRIES from .observation import ( MarkingError, @@ -332,6 +333,13 @@ def override_property(task, env, property): chat_size=(500, max(viewport["height"], 800)), record_video_dir=self.record_video_dir, ) + + # create the hint labeling ui + self.hint_labeling = HintLabeling( + headless=self.headless, + window_size=(500, max(viewport["height"], 800)), + record_video_dir=self.record_video_dir, + ) # create a new page self.page = self.context.new_page() diff --git a/browsergym/core/src/browsergym/core/hint_labeling.py b/browsergym/core/src/browsergym/core/hint_labeling.py new file mode 100644 index 000000000..2b2d4e0a4 --- /dev/null +++ b/browsergym/core/src/browsergym/core/hint_labeling.py @@ -0,0 +1,28 @@ +import playwright.sync_api + +from importlib import resources + +from . import _get_global_playwright, hint_labeling_files + +HINT_LABELING_DIR = resources.files(hint_labeling_files) + +class HintLabeling: + def __init__(self, headless: bool, window_size=(500, 800), *args, **kwargs): + + pw: playwright.sync_api.Playwright = _get_global_playwright() + self.browser = pw.chromium.launch( + headless=headless, args=[f"--window-size={window_size[0]},{window_size[1]}"] + ) + self.context = self.browser.new_context( + no_viewport=True, + ) + self.page = self.context.new_page() + + self.page.set_content(get_hint_labeling_ui(HINT_LABELING_DIR)) + + + +def get_hint_labeling_ui(hint_labeling_dir) -> str: + with open(hint_labeling_dir / "hint_labeling_ui.html", "r") as file: + hint_labeling_html = file.read() + return hint_labeling_html \ No newline at end of file diff --git a/browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html b/browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html new file mode 100644 index 000000000..23f7c2774 --- /dev/null +++ b/browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html @@ -0,0 +1,313 @@ + + + + + + Agent Reprompt UI + + + +
+ +
+
+

Goal

+
+
+
+

Error Feedback

+
+
+
+ + +
+
+ + + +
+
+
+ screenshot +
+ + +
+
+ + +
+

Hints

+ + + +
+ + +
+

Suggestions

+
+ +
+ + +
+ +
+
+ + + + diff --git a/browsergym/core/src/browsergym/utils/obs.py b/browsergym/core/src/browsergym/utils/obs.py index db5b8ce03..0abeb30e4 100644 --- a/browsergym/core/src/browsergym/utils/obs.py +++ b/browsergym/core/src/browsergym/utils/obs.py @@ -320,6 +320,13 @@ def dfs(node_idx: int, depth: int, parent_node_filtered: bool, parent_node_name: pass else: node_name = node["name"]["value"] + # if node_name == "": + # try: + # node_name_sources = node["name"]["sources"] + # node_name_sources = [elem for elem in node_name_sources if elem.get("type") == "contents"][0] + # node_name = node_name_sources.get("value", {}).get("value", "") + # except Exception as e: + # logger.warning(f"Failed to extract node name: {e}") if "value" in node and "value" in node["value"]: node_value = node["value"]["value"] else: diff --git a/browsergym/experiments/src/browsergym/experiments/loop.py b/browsergym/experiments/src/browsergym/experiments/loop.py index 0ebb9e94c..9ed20fdde 100644 --- a/browsergym/experiments/src/browsergym/experiments/loop.py +++ b/browsergym/experiments/src/browsergym/experiments/loop.py @@ -17,16 +17,15 @@ from pathlib import Path from typing import Optional -from browsergym.core.env import BrowserEnv import gymnasium as gym import numpy as np +from browsergym.core.action.parsers import highlevel_action_parser +from browsergym.core.chat import Chat +from browsergym.core.env import BrowserEnv from dataclasses_json import DataClassJsonMixin from PIL import Image from tqdm import tqdm -from browsergym.core.action.parsers import highlevel_action_parser -from browsergym.core.chat import Chat - from .agent import Agent from .utils import count_messages_token, count_tokens From ae4fe73308ab0df9c53be4229ce919e09c76d8c1 Mon Sep 17 00:00:00 2001 From: Patrice Bechard Date: Thu, 28 Aug 2025 00:24:19 -0400 Subject: [PATCH 2/8] update HintLabeling UI --- .../core/src/browsergym/core/hint_labeling.py | 124 +++++++++++++++++- .../hint_labeling_files/hint_labeling_ui.html | 47 +++++-- 2 files changed, 158 insertions(+), 13 deletions(-) diff --git a/browsergym/core/src/browsergym/core/hint_labeling.py b/browsergym/core/src/browsergym/core/hint_labeling.py index 2b2d4e0a4..a576099f6 100644 --- a/browsergym/core/src/browsergym/core/hint_labeling.py +++ b/browsergym/core/src/browsergym/core/hint_labeling.py @@ -1,11 +1,31 @@ -import playwright.sync_api - from importlib import resources +from typing import Dict, List, Optional +from queue import Queue, Empty +import playwright.sync_api +from pydantic import BaseModel, Field +import logging +import json from . import _get_global_playwright, hint_labeling_files +logger = logging.getLogger(__name__) + HINT_LABELING_DIR = resources.files(hint_labeling_files) +# ------- Data Classes ------- + +class HintLabelingInputs(BaseModel): + goal: str + error_feedback: str = "" + screenshot: str # base64 screenshot + axtree: str + history: List[Dict[str, str]] = Field(default_factory=list) + hint: str = "" + # keep 'suggestions' on Python side, but we’ll map to UI 'action_suggestions' + suggestions: List[Dict[str, str]] = Field(default_factory=list) + +# ------- Hint Labeling backend class ------- + class HintLabeling: def __init__(self, headless: bool, window_size=(500, 800), *args, **kwargs): @@ -17,10 +37,110 @@ def __init__(self, headless: bool, window_size=(500, 800), *args, **kwargs): no_viewport=True, ) self.page = self.context.new_page() + self._resp_queue: "Queue[dict]" = Queue() + self.page.route("**/api/reprompt", self._route_reprompt) + self.page.route("**/api/submit", self._route_submit) self.page.set_content(get_hint_labeling_ui(HINT_LABELING_DIR)) + # internal state + self._context: HintLabelingInputs = None + self._running = False + + def _route_reprompt(self, route: playwright.sync_api.Route, request: playwright.sync_api.Request): + logger.info("Route hit: %s %s", request.method, request.url) + try: + body = json.loads(request.post_data() or "{}") + except Exception: + body = {} + # enqueue output 1 (reprompt) + msg = {"type": "reprompt", "payload": {"hint": body.get("hint", "")}} + self._resp_queue.put(msg) + # Respond something minimal so UI doesn’t break; it will be refreshed by a later update_context() + route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"action_suggestions": []}), + ) + + def _route_submit(self, route: playwright.sync_api.Route, request: playwright.sync_api.Request): + logger.info("Route hit: %s %s", request.method, request.url) + try: + body = json.loads(request.post_data() or "{}") + except Exception: + body = {} + # Map UI payload -> your step shape + msg = { + "type": "step", + "payload": { + "think": body.get("think", ""), + "action": body.get("action", ""), + }, + } + self._resp_queue.put(msg) + # UI expects 200 JSON; we can optionally send new suggestions here too. + route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"action_suggestions": []}), + ) + + def _to_ui_bootstrap(self, ctx: HintLabelingInputs) -> dict: + # Map 'suggestions' [{action, thought}] -> 'action_suggestions' [{action, think}] + action_suggestions = [ + {"id": str(i + 1), "action": s.get("action", ""), "think": s.get("think", "")} + for i, s in enumerate(ctx.suggestions or []) + ] + return { + "goal": ctx.goal, + "error_feedback": ctx.error_feedback, + "screenshot": ctx.screenshot, + "axtree": ctx.axtree, + "history": ctx.history, + "hint": ctx.hint, + "action_suggestions": action_suggestions, + } + + def update_context(self, context: HintLabelingInputs): + self._context = context + ui_payload = self._to_ui_bootstrap(context) + # call JS function with arg (no string concat) + self.page.evaluate("updateContext", ui_payload) + + def wait_for_response(self, timeout: Optional[float] = 600) -> dict: + """ + Wait until the page makes a request to /api/reprompt or /api/submit, + then parse the request body and return it in your schema. + """ + logger.info("Waiting for response from Hint Labeling UI...") + + def is_api(req: playwright.sync_api.Request) -> bool: + u = req.url + return (u.endswith("/api/reprompt") or u.endswith("/api/submit")) and req.method == "POST" + + # This pumps Playwright internally; no busy waiting. + with self.page.expect_request(is_api, timeout=(timeout * 1000 if timeout else 0)) as req_info: + req = req_info.value + + body_text = req.post_data or "{}" + try: + body = json.loads(body_text) + except Exception as e: + print("JSON parse error:", e) + body = {} + + if req.url.endswith("/api/reprompt"): + msg = {"type": "reprompt", "payload": {"hint": body.get("hint", "")}} + else: + msg = {"type": "step", + "payload": {"think": body.get("think", ""), "action": body.get("action", "")}} + + logger.info("Response received: %s", msg) + return msg + def close(self): + self.context.close() + self.browser.close() def get_hint_labeling_ui(hint_labeling_dir) -> str: with open(hint_labeling_dir / "hint_labeling_ui.html", "r") as file: diff --git a/browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html b/browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html index 23f7c2774..889bd0cce 100644 --- a/browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html +++ b/browsergym/core/src/browsergym/core/hint_labeling_files/hint_labeling_ui.html @@ -3,6 +3,7 @@ + Agent Reprompt UI @@ -85,7 +85,7 @@

Error Feedback

screenshot