Harden Playwright routing/cancellation and bump to 0.2.4

Author: ImYourBoyRoy

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "web-scraper-toolkit"
-version = "0.2.3"
+version = "0.2.4"
 description = "A powerful, standalone web scraping toolkit using Playwright and various parsers."
 readme = "README.md"
 requires-python = ">=3.10"

src/web_scraper_toolkit/__init__.py

@@ -11,7 +11,7 @@
 
 """
 
-__version__ = "0.2.3"
+__version__ = "0.2.4"
 
 # Configs (Modular)
 from .browser.config import BrowserConfig
@@ -37,6 +37,7 @@
 from .parsers.scraping_tools import (
     read_website_markdown,
     read_website_content,
+    aread_website_markdown,
     capture_screenshot,
     save_as_pdf,
     extract_metadata,
@@ -71,6 +72,7 @@
     "smart_discover_urls",
     "read_website_markdown",
     "read_website_content",
+    "aread_website_markdown",
     "capture_screenshot",
     "save_as_pdf",
     "extract_metadata",

src/web_scraper_toolkit/browser/config.py

@@ -17,6 +17,7 @@
 class BrowserConfig:
     headless: bool = True
     browser_type: str = "chromium"
+    stealth_mode: bool = True
     viewport_width: int = 1280
     viewport_height: int = 800
     timeout: int = 30000

src/web_scraper_toolkit/browser/playwright_handler.py

@@ -40,7 +40,18 @@
 
 from .config import BrowserConfig
 
+try:
+    from playwright_stealth import Stealth as _StealthClass  # type: ignore[import-untyped]
+except ImportError:  # pragma: no cover
+    _StealthClass = None
+
+try:
+    from playwright_stealth import stealth_async as _legacy_stealth_async  # type: ignore[import-untyped]
+except ImportError:  # pragma: no cover - older/newer API compatibility
+    _legacy_stealth_async = None
+
 logger = logging.getLogger(__name__)
+_STEALTH_BACKEND_LOGGED = False
 
 # --- OPTIMIZED CONFIGURATION ---
 DEFAULT_USER_AGENTS = [
@@ -94,6 +105,7 @@ def __init__(
 
         self.browser_type_name = self.config.browser_type.lower()
         self.headless = self.config.headless
+        self.stealth_mode = bool(getattr(self.config, "stealth_mode", True))
 
         # Mapping properties
         self.user_agents = DEFAULT_USER_AGENTS
@@ -111,12 +123,64 @@ def __init__(
 
         self._playwright: Optional[Playwright] = None
         self._browser: Optional[Browser] = None
+        self._logged_socks_auth_skips: set[str] = set()
+        self._stealth = _StealthClass() if _StealthClass is not None else None
+        self._stealth_missing_warned = False
+        global _STEALTH_BACKEND_LOGGED
+        if not _STEALTH_BACKEND_LOGGED:
+            if self._stealth is not None:
+                logger.info("Playwright stealth backend: playwright_stealth.Stealth")
+            elif callable(_legacy_stealth_async):
+                logger.info(
+                    "Playwright stealth backend: playwright_stealth.stealth_async"
+                )
+            else:
+                logger.info(
+                    "Playwright stealth backend: unavailable (basic webdriver scrub only)"
+                )
+            _STEALTH_BACKEND_LOGGED = True
 
         logger.info(
             f"PlaywrightManager initialized: Browser={self.browser_type_name}, Headless={self.headless}, "
             f"Default Timeout={self.default_navigation_timeout_ms}ms"
         )
 
+    def _build_playwright_proxy_settings(self, proxy_obj: Any) -> Dict[str, str]:
+        """
+        Build Playwright proxy settings from a Proxy model.
+        NOTE: Playwright does not support SOCKS authentication fields.
+        """
+        protocol = (
+            proxy_obj.protocol.value
+            if hasattr(proxy_obj.protocol, "value")
+            else str(proxy_obj.protocol)
+        ).lower()
+        proxy_settings: Dict[str, str] = {
+            "server": f"{protocol}://{proxy_obj.hostname}:{proxy_obj.port}"
+        }
+
+        username = str(getattr(proxy_obj, "username", "") or "")
+        password = str(getattr(proxy_obj, "password", "") or "")
+        has_auth = bool(username or password)
+        supports_auth = protocol in {"http", "https"}
+
+        if supports_auth:
+            if username:
+                proxy_settings["username"] = username
+            if password:
+                proxy_settings["password"] = password
+        elif has_auth:
+            warning_key = f"{proxy_obj.hostname}:{proxy_obj.port}:{protocol}"
+            if warning_key not in self._logged_socks_auth_skips:
+                logger.warning(
+                    "Proxy %s provided SOCKS credentials, but Playwright does not "
+                    "support SOCKS authentication. Credentials will be ignored.",
+                    warning_key,
+                )
+                self._logged_socks_auth_skips.add(warning_key)
+
+        return proxy_settings
+
     @property
     def browser(self) -> Browser:
         """Expose active browser for backward-compatible advanced operations."""
@@ -202,21 +266,12 @@ async def get_new_page(
                     # Construct Playwright Proxy Dict
                     # protocol://user:pass@host:port OR separate fields
                     # Playwright expects: { "server": "...", "username": "...", "password": "..." }
-
-                    # Protocol handling
                     protocol = (
                         proxy_obj.protocol.value
                         if hasattr(proxy_obj.protocol, "value")
                         else str(proxy_obj.protocol)
                     )
-
-                    proxy_settings = {
-                        "server": f"{protocol}://{proxy_obj.hostname}:{proxy_obj.port}"
-                    }
-                    if proxy_obj.username:
-                        proxy_settings["username"] = proxy_obj.username
-                    if proxy_obj.password:
-                        proxy_settings["password"] = proxy_obj.password
+                    proxy_settings = self._build_playwright_proxy_settings(proxy_obj)
 
                     base_context_options["proxy"] = proxy_settings
                     logger.info(
@@ -239,16 +294,25 @@ async def get_new_page(
             )
 
             # Tracker and ad blocking for faster, cleaner page loads
-            await context.route(
-                "**/*",
-                lambda route: (
-                    route.abort()
-                    if self._is_tracker_or_ad(route.request.url)
-                    else route.continue_()
-                ),
-            )
+            async def _route_handler(route: Any) -> None:
+                if self._is_tracker_or_ad(route.request.url):
+                    await route.abort()
+                    return
+                await route.continue_()
+
+            await context.route("**/*", _route_handler)
 
             page = await context.new_page()
+            if self.stealth_mode and self._stealth is not None:
+                await self._stealth.apply_stealth_async(page)
+            elif self.stealth_mode and callable(_legacy_stealth_async):
+                await _legacy_stealth_async(page)
+            elif self.stealth_mode and not self._stealth_missing_warned:
+                logger.warning(
+                    "stealth_mode is enabled but playwright_stealth is unavailable; "
+                    "falling back to basic webdriver scrubbing only."
+                )
+                self._stealth_missing_warned = True
             return page, context
         except Exception as e:
             logger.error(f"Error creating new page and context: {e}", exc_info=True)
@@ -404,6 +468,14 @@ async def fetch_page_content(
                     f"Playwright: Timeout on {current_url_val} "
                     f"(attempt {attempt + 1}/{effective_retries + 1}): {pte}"
                 )
+            except asyncio.CancelledError:
+                logger.warning(
+                    "Playwright: Fetch cancelled on %s (attempt %s/%s).",
+                    current_url_val,
+                    attempt + 1,
+                    effective_retries + 1,
+                )
+                raise
             except Exception as e:
                 logger.error(
                     f"Playwright: Unexpected error on {current_url_val} "
@@ -493,6 +565,9 @@ async def smart_fetch(
                     )
 
             return content, final_url, status
+        except asyncio.CancelledError:
+            logger.warning("SmartFetch cancelled for %s.", url)
+            raise
 
         finally:
             if page:

src/web_scraper_toolkit/parsers/content.py

@@ -56,15 +56,17 @@ async def _arun_scrape(
     """Async helper for scraping."""
     manager = None
     # Config handling
-    # Config handling
     browser_cfg = BrowserConfig()  # default
     if isinstance(config, BrowserConfig):
         browser_cfg = config
     elif isinstance(config, dict):
-        # Convert dict to BrowserConfig
         browser_cfg = BrowserConfig(
             headless=config.get("headless", True),
             browser_type=config.get("browser_type", "chromium"),
+            stealth_mode=config.get("stealth_mode", True),
+            viewport_width=config.get("viewport_width", 1280),
+            viewport_height=config.get("viewport_height", 800),
+            timeout=config.get("timeout", 30000),
         )
 
     try:
@@ -154,28 +156,54 @@ def read_website_content(
     return _run_coro_sync(_arun_scrape(website_url, config))
 
 
-async def _arun_scrape_markdown(
+def _dict_to_browser_config(config: Dict[str, Any]) -> "BrowserConfig":
+    """Convert a dict to BrowserConfig, preserving all supported fields."""
+    return BrowserConfig(
+        headless=config.get("headless", True),
+        browser_type=config.get("browser_type", "chromium"),
+        stealth_mode=config.get("stealth_mode", True),
+        viewport_width=config.get("viewport_width", 1280),
+        viewport_height=config.get("viewport_height", 800),
+        timeout=config.get("timeout", 30000),
+    )
+
+
+async def aread_website_markdown(
     website_url: str,
     config: Optional[Union[Dict[str, Any], ParserConfig, BrowserConfig]] = None,
     selector: Optional[str] = None,
     max_length: Optional[int] = None,
+    playwright_manager: Optional[Any] = None,
 ) -> str:
-    """Async helper for scraping and converting to Markdown."""
-    manager = None
+    """Async version of read_website_markdown.
+
+    Fetches a website and converts the content to clean Markdown.
+
+    Args:
+        website_url: The full URL of the website to read.
+        config: Browser configuration (dict, ParserConfig, or BrowserConfig).
+        selector: Optional CSS selector to extract only specific content.
+        max_length: Optional character limit for the output.
+        playwright_manager: Optional pre-started PlaywrightManager instance
+            for browser session reuse. When provided, the caller owns the
+            lifecycle — this function will NOT start or stop the manager.
+    """
+    owns_manager = playwright_manager is None
+    manager = playwright_manager
+
     browser_cfg = BrowserConfig()
     if isinstance(config, BrowserConfig):
         browser_cfg = config
     elif isinstance(config, dict):
-        browser_cfg = BrowserConfig(
-            headless=config.get("headless", True),
-            browser_type=config.get("browser_type", "chromium"),
-            timeout=config.get("timeout", 30000),
-        )
+        browser_cfg = _dict_to_browser_config(config)
+
     try:
-        from ..browser.playwright_handler import PlaywrightManager
+        if manager is None:
+            from ..browser.playwright_handler import PlaywrightManager
+
+            manager = PlaywrightManager(config=browser_cfg)
+            await manager.start()
 
-        manager = PlaywrightManager(config=browser_cfg)
-        await manager.start()
         # Use Smart Fetch for robustness
         content, final_url, status_code = await manager.smart_fetch(url=website_url)
 
@@ -213,10 +241,14 @@ async def _arun_scrape_markdown(
         )
         return f"An error occurred while scraping the website: {str(e)}"
     finally:
-        if manager:
+        if owns_manager and manager:
             await manager.stop()
 
 
+# Backward-compatible alias
+_arun_scrape_markdown = aread_website_markdown
+
+
 def read_website_markdown(
     website_url: str,
     config: Optional[Union[Dict[str, Any], ParserConfig, BrowserConfig]] = None,
@@ -235,5 +267,5 @@ def read_website_markdown(
     """
     logger.info(f"Executing read_website_markdown for URL: {website_url}")
     return _run_coro_sync(
-        _arun_scrape_markdown(website_url, config, selector, max_length)
+        aread_website_markdown(website_url, config, selector, max_length)
     )