[py] improve how tests are run against the grid server (#16851)

* [py] improve how tests are run against the grid server

* put the test-remote job back as alias

* use removesuffix

Author: titusfortner

py/BUILD.bazel

@@ -82,6 +82,14 @@ TEST_DEPS = [
     "@rules_python//python/runfiles",
 ]
 
+genrule(
+    name = "java-location",
+    srcs = [],
+    outs = ["java-location.txt"],
+    cmd = "echo $(JAVA) > $@",
+    toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"],
+)
+
 copy_file(
     name = "manager-linux",
     src = "//common/manager:selenium-manager-linux",
@@ -502,33 +510,57 @@ BROWSER_TESTS = {
     if BROWSER_TESTS[browser].get("bidi", False)
 ]
 
-py_test_suite(
-    name = "test-remote",
-    size = "large",
-    srcs = glob(
-        [
-            "test/selenium/webdriver/common/**/*.py",
-            "test/selenium/webdriver/remote/**/*.py",
-            "test/selenium/webdriver/support/**/*.py",
+# Generate test-<browser>-remote targets (chrome and firefox only)
+[
+    py_test_suite(
+        name = "test-%s-remote" % browser,
+        size = "large",
+        srcs = glob(
+            [
+                "test/selenium/webdriver/common/**/*.py",
+                "test/selenium/webdriver/remote/**/*.py",
+                "test/selenium/webdriver/support/**/*.py",
+            ] + BROWSER_TESTS[browser]["browser_srcs"],
+            exclude = BIDI_TESTS + ["test/selenium/webdriver/common/print_pdf_tests.py"] +
+                      BROWSER_TESTS[browser].get("extra_excludes", []),
+        ),
+        args = [
+            "--instafail",
+            "--remote",
+        ] + BROWSERS[browser]["args"],
+        data = BROWSERS[browser]["data"] + [
+            ":java-location",
+            "//java/src/org/openqa/selenium/grid:selenium_server_deploy.jar",
+            "@bazel_tools//tools/jdk:current_java_runtime",
         ],
-        exclude = BIDI_TESTS,
-    ),
-    args = [
-        "--instafail",
-        "--driver=remote",
-    ],
-    data = [
-        "//java/src/org/openqa/selenium/grid:selenium_server_deploy.jar",
-    ],
-    tags = [
-        "no-sandbox",
-        "skip-rbe",
+        env = {
+            "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)",
+        },
+        env_inherit = ["DISPLAY"],
+        tags = [
+            "no-sandbox",
+            "remote",
+            "%s-remote" % browser,
+        ],
+        deps = [
+            ":init-tree",
+            ":selenium",
+            ":webserver",
+        ] + TEST_DEPS,
+    )
+    for browser in [
+        "chrome",
+        "firefox",
+    ]
+]
+
+test_suite(
+    name = "test-remote",
+    tags = ["remote"],
+    tests = [
+        ":test-chrome-remote",
+        ":test-firefox-remote",
     ],
-    deps = [
-        ":init-tree",
-        ":selenium",
-        ":webserver",
-    ] + TEST_DEPS,
 )
 
 py_test_suite(

py/conftest.py

@@ -27,6 +27,7 @@
 import pytest
 import rich.console
 import rich.traceback
+from python.runfiles import Runfiles
 
 from selenium import webdriver
 from selenium.common.exceptions import WebDriverException
@@ -40,7 +41,6 @@
     "edge",
     "firefox",
     "ie",
-    "remote",
     "safari",
     "webkitgtk",
     "wpewebkit",
@@ -150,6 +150,12 @@ def pytest_addoption(parser):
         dest="bidi",
         help="Enable BiDi support",
     )
+    parser.addoption(
+        "--remote",
+        action="store_true",
+        dest="remote",
+        help="Run tests against a remote Grid server",
+    )
 
 
 def pytest_ignore_collect(collection_path, config):
@@ -186,7 +192,6 @@ class SupportedDrivers(ContainerProtocol):
     ie: str = "Ie"
     webkitgtk: str = "WebKitGTK"
     wpewebkit: str = "WPEWebKit"
-    remote: str = "Remote"
 
 
 @dataclass
@@ -196,7 +201,6 @@ class SupportedOptions(ContainerProtocol):
     edge: str = "EdgeOptions"
     safari: str = "SafariOptions"
     ie: str = "IeOptions"
-    remote: str = "ChromeOptions"
     webkitgtk: str = "WebKitGTKOptions"
     wpewebkit: str = "WPEWebKitOptions"
 
@@ -206,7 +210,6 @@ class SupportedBidiDrivers(ContainerProtocol):
     chrome: str = "Chrome"
     firefox: str = "Firefox"
     edge: str = "Edge"
-    remote: str = "Remote"
 
 
 class Driver:
@@ -215,6 +218,7 @@ def __init__(self, driver_class, request):
         self._request = request
         self._driver = None
         self._service = None
+        self._server = None
         self.options = driver_class
         self.headless = driver_class
         self.bidi = driver_class
@@ -313,17 +317,16 @@ def options(self, cls_name):
                 # There are issues with window size/position when running Firefox
                 # under Wayland, so we use XWayland instead.
                 os.environ["MOZ_ENABLE_WAYLAND"] = "0"
-        elif self.driver_class == self.supported_drivers.remote:
-            self._options = getattr(webdriver, self.supported_options.chrome)()
-            self._options.set_capability("goog:chromeOptions", {})
-            self._options.enable_downloads = True
         else:
             opts_cls = getattr(self.supported_options, cls_name.lower())
             self._options = getattr(webdriver, opts_cls)()
 
         if cls_name.lower() in ("chrome", "edge"):
             self._options.add_argument("--disable-dev-shm-usage")
 
+        if self.is_remote:
+            self._options.enable_downloads = True
+
         if self.browser_path or self.browser_args:
             if self.driver_class == self.supported_drivers.webkitgtk:
                 self._options.overlay_scrollbars_enabled = False
@@ -358,10 +361,17 @@ def is_platform_valid(self):
             return False
         return True
 
+    @property
+    def is_remote(self):
+        return self._request.config.getoption("remote")
+
     def _initialize_driver(self):
         kwargs = {}
         if self.options is not None:
             kwargs["options"] = self.options
+        if self.is_remote:
+            kwargs["command_executor"] = self._server.status_url.removesuffix("/status")
+            return webdriver.Remote(**kwargs)
         if self.driver_path is not None:
             kwargs["service"] = self.service
         return getattr(webdriver, self.driver_class)(**kwargs)
@@ -374,20 +384,22 @@ def stop_driver(self):
 
 
 @pytest.fixture
-def driver(request):
+def driver(request, server):
     global selenium_driver
     driver_class = getattr(request, "param", "Chrome").lower()
 
     if selenium_driver is None:
         selenium_driver = Driver(driver_class, request)
+    if server:
+        selenium_driver._server = server
 
     # skip tests if not available on the platform
     if not selenium_driver.is_platform_valid:
         pytest.skip(f"{driver_class} tests can only run on {selenium_driver.exe_platform}")
 
-    # skip tests in the 'remote' directory if run with a local driver
-    if request.node.path.parts[-2] == "remote" and selenium_driver.driver_class != "Remote":
-        pytest.skip(f"Remote tests can't be run with driver '{selenium_driver.driver_class}'")
+    # skip tests in the 'remote' directory if not running with --remote flag
+    if request.node.path.parts[-2] == "remote" and not selenium_driver.is_remote:
+        pytest.skip("Remote tests require the --remote flag")
 
     # skip tests for drivers that don't support BiDi when --bidi is enabled
     if selenium_driver.bidi:
@@ -396,17 +408,23 @@ def driver(request):
 
     # conditionally mark tests as expected to fail based on driver
     marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}")
+    # Also check for xfail_remote when running with --remote
+    if marker is None and selenium_driver.is_remote:
+        marker = request.node.get_closest_marker("xfail_remote")
     if marker is not None:
-        if "run" in marker.kwargs:
-            if not marker.kwargs["run"]:
-                pytest.skip()
-                yield
-                return
-        if "raises" in marker.kwargs:
-            marker.kwargs.pop("raises")
-        pytest.xfail(**marker.kwargs)
-
-        request.addfinalizer(selenium_driver.stop_driver)
+        kwargs = dict(marker.kwargs)
+        # Support condition kwarg - if condition is False, skip the xfail
+        condition = kwargs.pop("condition", True)
+        if callable(condition):
+            condition = condition()
+        if condition:
+            if "run" in kwargs:
+                if not kwargs["run"]:
+                    pytest.skip()
+                    yield
+                    return
+            kwargs.pop("raises", None)
+            pytest.xfail(**kwargs)
 
     # For BiDi tests, only restart driver when explicitly marked as needing fresh driver.
     # Tests marked with @pytest.mark.needs_fresh_driver get full driver restart for test isolation.
@@ -477,28 +495,38 @@ def load(self, name):
 
 @pytest.fixture(autouse=True, scope="session")
 def server(request):
-    drivers = request.config.getoption("drivers")
-    if drivers is None or "remote" not in drivers:
+    is_remote = request.config.getoption("remote")
+    if not is_remote:
         yield None
         return
 
-    jar_path = os.path.join(
-        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-        "java/src/org/openqa/selenium/grid/selenium_server_deploy.jar",
-    )
+    r = Runfiles.Create()
+
+    java_location_txt = r.Rlocation("_main/" + os.environ.get("SE_BAZEL_JAVA_LOCATION"))
+    try:
+        with open(java_location_txt, encoding="utf-8") as handle:
+            read = handle.read().strip()
+            rel_path = read[len("external/") :] if read.startswith("external/") else read
+            java_path = r.Rlocation(rel_path)
+    except Exception:
+        java_path = None
+
+    built_jar = "selenium/java/src/org/openqa/selenium/grid/selenium_server_deploy.jar"
+    jar_path = r.Rlocation(built_jar)
 
     remote_env = os.environ.copy()
     if sys.platform == "linux":
         # There are issues with window size/position when running Firefox
         # under Wayland, so we use XWayland instead.
         remote_env["MOZ_ENABLE_WAYLAND"] = "0"
 
+    server = Server(env=remote_env, startup_timeout=60)
+    if Path(java_path).exists():
+        server.java_path = java_path
     if Path(jar_path).exists():
-        # use the grid server built by bazel
-        server = Server(path=jar_path, env=remote_env)
-    else:
-        # use the local grid server (downloads a new one if needed)
-        server = Server(env=remote_env)
+        server.path = jar_path
+
+    server.port = free_port()
     server.start()
     yield server
     server.stop()
@@ -537,15 +565,18 @@ def clean_driver(request):
 
     # conditionally mark tests as expected to fail based on driver
     marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}")
+    # Also check for xfail_remote when running with --remote
+    if marker is None and request.config.getoption("remote"):
+        marker = request.node.get_closest_marker("xfail_remote")
     if marker is not None:
-        if "run" in marker.kwargs:
-            if not marker.kwargs["run"]:
+        kwargs = dict(marker.kwargs)
+        if "run" in kwargs:
+            if not kwargs["run"]:
                 pytest.skip()
                 yield
                 return
-        if "raises" in marker.kwargs:
-            marker.kwargs.pop("raises")
-        pytest.xfail(**marker.kwargs)
+        kwargs.pop("raises", None)
+        pytest.xfail(**kwargs)
 
     yield driver_reference
 
@@ -568,19 +599,19 @@ def clean_options(request):
 
 @pytest.fixture
 def firefox_options(request):
-    _supported_drivers = SupportedDrivers()
     try:
         driver_class = request.config.option.drivers[0].lower()
     except (AttributeError, TypeError):
         raise Exception("This test requires a --driver to be specified")
 
-    # skip if not Firefox or Remote
-    if driver_class not in ("firefox", "remote"):
-        pytest.skip(f"This test requires Firefox or Remote. Got {driver_class}")
+    # skip if not Firefox
+    if driver_class != "firefox":
+        pytest.skip(f"This test requires Firefox. Got {driver_class}")
 
-    # skip tests in the 'remote' directory if run with a local driver
-    if request.node.path.parts[-2] == "remote" and getattr(_supported_drivers, driver_class) != "Remote":
-        pytest.skip(f"Remote tests can't be run with driver '{driver_class}'")
+    # skip tests in the 'remote' directory if not running with --remote flag
+    is_remote = request.config.getoption("remote")
+    if request.node.path.parts[-2] == "remote" and not is_remote:
+        pytest.skip("Remote tests require the --remote flag")
 
     options = Driver.clean_options("firefox", request)
 
@@ -589,24 +620,21 @@ def firefox_options(request):
 
 @pytest.fixture
 def chromium_options(request):
-    _supported_drivers = SupportedDrivers()
     try:
         driver_class = request.config.option.drivers[0].lower()
     except (AttributeError, TypeError):
         raise Exception("This test requires a --driver to be specified")
 
-    # skip if not Chrome, Edge, or Remote
-    if driver_class not in ("chrome", "edge", "remote"):
-        pytest.skip(f"This test requires Chrome, Edge, or Remote. Got {driver_class}")
+    # skip if not Chrome or Edge
+    if driver_class not in ("chrome", "edge"):
+        pytest.skip(f"This test requires Chrome or Edge. Got {driver_class}")
 
-    # skip tests in the 'remote' directory if run with a local driver
-    if request.node.path.parts[-2] == "remote" and getattr(_supported_drivers, driver_class) != "Remote":
-        pytest.skip(f"Remote tests can't be run with driver '{driver_class}'")
+    # skip tests in the 'remote' directory if not running with --remote flag
+    is_remote = request.config.getoption("remote")
+    if request.node.path.parts[-2] == "remote" and not is_remote:
+        pytest.skip("Remote tests require the --remote flag")
 
-    if driver_class in ("chrome", "remote"):
-        options = Driver.clean_options("chrome", request)
-    else:
-        options = Driver.clean_options("edge", request)
+    options = Driver.clean_options(driver_class, request)
 
     return options