feat: get rid of rest_host, realtime_host internally unified everything under host

Author: ttypic

ably/http/http.py

@@ -140,9 +140,9 @@ def dump_body(self, body):
         else:
             return json.dumps(body, separators=(',', ':'))
 
-    def get_rest_hosts(self):
-        hosts = self.options.get_rest_hosts()
-        host = self.__host or self.options.fallback_realtime_host
+    def get_hosts(self):
+        hosts = self.options.get_hosts()
+        host = self.__host
         if host is None:
             return hosts
 
@@ -186,7 +186,7 @@ async def make_request(self, method, path, version=None, headers=None, body=None
         http_max_retry_duration = self.http_max_retry_duration
         requested_at = time.time()
 
-        hosts = self.get_rest_hosts()
+        hosts = self.get_hosts()
         for retry_count, host in enumerate(hosts):
             def should_stop_retrying(retry_count=retry_count):
                 time_passed = time.time() - requested_at
@@ -229,7 +229,7 @@ def should_stop_retrying(retry_count=retry_count):
                         continue
 
                     # Keep fallback host for later (RSC15f)
-                    if retry_count > 0 and host != self.options.get_rest_host():
+                    if retry_count > 0 and host != self.options.get_host():
                         self.__host = host
                         self.__host_expires = time.time() + (self.options.fallback_retry_timeout / 1000.0)
 
@@ -277,7 +277,7 @@ def options(self):
 
     @property
     def preferred_host(self):
-        return self.options.get_rest_host()
+        return self.options.get_host()
 
     @property
     def preferred_port(self):

ably/realtime/connectionmanager.py

@@ -122,7 +122,7 @@ def __init__(self, realtime: AblyRealtime, initial_state):
         self.retry_timer: Timer | None = None
         self.connect_base_task: asyncio.Task | None = None
         self.disconnect_transport_task: asyncio.Task | None = None
-        self.__fallback_hosts: list[str] = self.options.get_fallback_realtime_hosts()
+        self.__fallback_hosts: list[str] = self.options.get_fallback_hosts()
         self.queued_messages: deque[PendingMessage] = deque()
         self.__error_reason: AblyException | None = None
         self.msg_serial: int = 0
@@ -532,7 +532,7 @@ async def connect_with_fallback_hosts(self, fallback_hosts: list) -> Exception |
 
     async def connect_base(self) -> None:
         fallback_hosts = self.__fallback_hosts
-        primary_host = self.options.get_realtime_host()
+        primary_host = self.options.get_host()
         try:
             await self.try_host(primary_host)
             return

ably/transport/websockettransport.py

@@ -142,8 +142,8 @@ async def on_protocol_message(self, msg):
                 self.max_idle_interval = max_idle_interval + self.options.realtime_request_timeout
                 self.on_activity()
             self.is_connected = True
-            if self.host != self.options.get_realtime_host():  # RTN17e
-                self.options.fallback_realtime_host = self.host
+            if self.host != self.options.get_host():  # RTN17e
+                self.options.fallback_host = self.host
             self.connection_manager.on_connected(connection_details, connection_id, reason=exception)
         elif action == ProtocolMessageAction.DISCONNECTED:
             error = msg.get('error')

ably/types/options.py

@@ -1,5 +1,4 @@
 import logging
-import random
 from abc import ABC, abstractmethod
 
 from ably.transport.defaults import Defaults
@@ -29,17 +28,21 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti
                  tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None,
                  environment=None, http_open_timeout=None, http_request_timeout=None,
                  realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None,
-                 fallback_hosts=None, fallback_retry_timeout=None, disconnected_retry_timeout=None,
-                 idempotent_rest_publishing=None, loop=None, auto_connect=True,
+                 fallback_hosts=None, fallback_hosts_use_default=None, fallback_retry_timeout=None,
+                 disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True,
                  suspended_retry_timeout=None, connectivity_check_url=None,
                  channel_retry_timeout=Defaults.channel_retry_timeout, add_request_ids=False,
                  vcdiff_decoder: VCDiffDecoder = None, transport_params=None, **kwargs):
 
         super().__init__(**kwargs)
 
+        # REC1b1: endpoint is incompatible with deprecated options
         if endpoint is not None:
-            if environment is not None or rest_host is not None or realtime_host is not None:
-                raise ValueError('endpoint is incompatible with any of environment, rest_host or realtime_host')
+            if (environment is not None or rest_host is not None or
+                    realtime_host is not None or fallback_hosts_use_default is not None):
+                raise ValueError(
+                    'endpoint is incompatible with any of environment, rest_host, '
+                    'realtime_host or fallback_hosts_use_default')
 
         # TODO check these defaults
         if fallback_retry_timeout is None:
@@ -65,21 +68,34 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti
         if environment is not None and realtime_host is not None:
             raise ValueError('specify realtime_host or environment, not both')
 
+        # REC2a1: fallback_hosts is incompatible with fallback_hosts_use_default
+        if fallback_hosts is not None and fallback_hosts_use_default is not None:
+            raise ValueError('fallback_hosts is incompatible with fallback_hosts_use_default')
+
         if idempotent_rest_publishing is None:
             from ably import api_version
             idempotent_rest_publishing = api_version >= '1.2'
 
         if environment is not None and endpoint is None:
+            log.warning("environment client option is deprecated, please use endpoint instead")
             endpoint = environment
 
+        # REC1d: restHost or realtimeHost option
+        # REC1d1: restHost takes precedence over realtimeHost
+        if rest_host is not None and endpoint is None:
+            log.warning("rest_host client option is deprecated, please use endpoint instead")
+            endpoint = rest_host
+        elif realtime_host is not None and endpoint is None:
+            # REC1d2: realtimeHost if restHost not specified
+            log.warning("realtime_host client option is deprecated, please use endpoint instead")
+            endpoint = realtime_host
+
         if endpoint is None:
             endpoint = Defaults.endpoint
 
         self.__client_id = client_id
         self.__log_level = log_level
         self.__tls = tls
-        self.__rest_host = rest_host
-        self.__realtime_host = realtime_host
         self.__port = port
         self.__tls_port = tls_port
         self.__use_binary_protocol = use_binary_protocol
@@ -92,6 +108,7 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti
         self.__http_max_retry_count = http_max_retry_count
         self.__http_max_retry_duration = http_max_retry_duration
         self.__fallback_hosts = fallback_hosts
+        self.__fallback_hosts_use_default = fallback_hosts_use_default
         self.__fallback_retry_timeout = fallback_retry_timeout
         self.__disconnected_retry_timeout = disconnected_retry_timeout
         self.__channel_retry_timeout = channel_retry_timeout
@@ -101,13 +118,10 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti
         self.__connection_state_ttl = connection_state_ttl
         self.__suspended_retry_timeout = suspended_retry_timeout
         self.__connectivity_check_url = connectivity_check_url
-        self.__fallback_realtime_host = None
         self.__add_request_ids = add_request_ids
         self.__vcdiff_decoder = vcdiff_decoder
         self.__transport_params = transport_params or {}
-
-        self.__rest_hosts = self.__get_rest_hosts()
-        self.__realtime_hosts = self.__get_realtime_hosts()
+        self.__hosts = self.__get_hosts()
 
     @property
     def client_id(self):
@@ -133,23 +147,6 @@ def tls(self):
     def tls(self, value):
         self.__tls = value
 
-    @property
-    def rest_host(self):
-        return self.__rest_host
-
-    @rest_host.setter
-    def rest_host(self, value):
-        self.__rest_host = value
-
-    # RTC1d
-    @property
-    def realtime_host(self):
-        return self.__realtime_host
-
-    @realtime_host.setter
-    def realtime_host(self, value):
-        self.__realtime_host = value
-
     @property
     def port(self):
         return self.__port
@@ -275,14 +272,6 @@ def suspended_retry_timeout(self):
     def connectivity_check_url(self):
         return self.__connectivity_check_url
 
-    @property
-    def fallback_realtime_host(self):
-        return self.__fallback_realtime_host
-
-    @fallback_realtime_host.setter
-    def fallback_realtime_host(self, value):
-        self.__fallback_realtime_host = value
-
     @property
     def add_request_ids(self):
         return self.__add_request_ids
@@ -295,61 +284,33 @@ def vcdiff_decoder(self):
     def transport_params(self):
         return self.__transport_params
 
-    def __get_rest_hosts(self):
-        """
-        Return the list of hosts as they should be tried. First comes the main
-        host. Then the fallback hosts in random order.
-        The returned list will have a length of up to http_max_retry_count.
-        """
-        # Defaults
-        host = self.rest_host
-        if host is None:
-            host = Defaults.get_hostname(self.endpoint)
-
-        http_max_retry_count = self.http_max_retry_count
-        if http_max_retry_count is None:
-            http_max_retry_count = Defaults.http_max_retry_count
-
-        # Fallback hosts
-        fallback_hosts = self.fallback_hosts
-        if fallback_hosts is None:
-            if self.rest_host is not None:
-                fallback_hosts = []
-            else:
-                fallback_hosts = Defaults.get_fallback_hosts(self.endpoint)
-
-        # Shuffle
-        fallback_hosts = list(fallback_hosts)
-        random.shuffle(fallback_hosts)
-        self.__fallback_hosts = fallback_hosts
-
-        # First main host
-        hosts = [host] + fallback_hosts
-        hosts = hosts[:http_max_retry_count]
-        return hosts
-
-    def __get_realtime_hosts(self):
-        if self.realtime_host is not None:
-            host = self.realtime_host
-            return [host]
-
+    def __get_hosts(self):
         host = Defaults.get_hostname(self.endpoint)
-        return [host] + self.__fallback_hosts
+        # REC2: Determine fallback hosts
+        fallback_hosts = self.get_fallback_hosts()
+        return [host] + fallback_hosts
 
-    def get_rest_hosts(self):
-        return self.__rest_hosts
+    def get_hosts(self):
+        return self.__hosts
 
-    def get_rest_host(self):
-        return self.__rest_hosts[0]
+    def get_host(self):
+        return self.__hosts[0]
 
-    def get_realtime_hosts(self):
-        return self.__realtime_hosts
+    # REC2: Various client options collectively determine a set of fallback domains
+    def get_fallback_hosts(self):
+        # REC2a: If the fallbackHosts client option is specified
+        if self.__fallback_hosts is not None:
+            # REC2a2: the set of fallback domains is given by the value of the fallbackHosts option
+            return self.__fallback_hosts
 
-    def get_realtime_host(self):
-        return self.__realtime_hosts[0]
+        # REC2b: Otherwise, if the deprecated fallbackHostsUseDefault option is specified
+        if self.__fallback_hosts_use_default is not None and self.__fallback_hosts_use_default:
+            # REC2b: then the set of fallback domains is the default set defined in (REC2c1)
+            return Defaults.get_fallback_hosts("main")
 
-    def get_fallback_rest_hosts(self):
-        return self.__rest_hosts[1:]
+        # REC2c: Otherwise, the set of fallback domains is defined implicitly by the options
+        # used to define the primary domain as specified in (REC1)
+        return Defaults.get_fallback_hosts(self.endpoint)
 
     def get_fallback_realtime_hosts(self):
-        return self.__realtime_hosts[1:]
+        return self.get_fallback_hosts()

test/ably/realtime/realtimeconnection_test.py

@@ -187,7 +187,7 @@ async def test_connectivity_check_bad_status(self):
         assert ably.connection.connection_manager.check_connection() is False
 
     async def test_unroutable_host(self):
-        ably = await TestApp.get_ably_realtime(realtime_host="10.255.255.1", realtime_request_timeout=3000)
+        ably = await TestApp.get_ably_realtime(endpoint="10.255.255.1", realtime_request_timeout=3000)
         state_change = await ably.connection.once_async()
         assert state_change.reason
         assert state_change.reason.code == 50003
@@ -197,7 +197,7 @@ async def test_unroutable_host(self):
         await ably.close()
 
     async def test_invalid_host(self):
-        ably = await TestApp.get_ably_realtime(realtime_host="iamnotahost")
+        ably = await TestApp.get_ably_realtime(endpoint="iamnotahost")
         state_change = await ably.connection.once_async()
         assert state_change.reason
         assert state_change.reason.code == 40000
@@ -299,8 +299,8 @@ async def test_fallback_host(self):
 
         await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5)
 
-        assert ably.connection.connection_manager.transport.host != self.test_vars["realtime_host"]
-        assert ably.options.fallback_realtime_host != self.test_vars["realtime_host"]
+        assert ably.connection.connection_manager.transport.host != self.test_vars["endpoint"]
+        assert ably.options.fallback_host != self.test_vars["endpoint"]
         await ably.close()
 
     async def test_fallback_host_no_connection(self):
@@ -325,7 +325,7 @@ def check_connection():
 
         await ably.connection.once_async(ConnectionState.DISCONNECTED)
 
-        assert ably.options.fallback_realtime_host is None
+        assert ably.options.fallback_host is None
         await ably.close()
 
     async def test_fallback_host_disconnected_protocol_msg(self):
@@ -344,8 +344,8 @@ async def test_fallback_host_disconnected_protocol_msg(self):
 
         await asyncio.wait_for(ably.connection.once_async(ConnectionState.CONNECTED), timeout=5)
 
-        assert ably.connection.connection_manager.transport.host != self.test_vars["realtime_host"]
-        assert ably.options.fallback_realtime_host != self.test_vars["realtime_host"]
+        assert ably.connection.connection_manager.transport.host != self.test_vars["endpoint"]
+        assert ably.options.fallback_host != self.test_vars["endpoint"]
         await ably.close()
 
     #  RTN2d

test/ably/rest/restauth_test.py

@@ -486,7 +486,7 @@ class TestRenewToken(BaseAsyncTestCase):
     async def setup(self):
         self.test_vars = await TestApp.get_test_vars()
         self.host = 'fake-host.ably.io'
-        self.ably = await TestApp.get_ably_rest(use_binary_protocol=False, rest_host=self.host)
+        self.ably = await TestApp.get_ably_rest(use_binary_protocol=False, endpoint=self.host)
         # with headers
         self.publish_attempts = 0
         self.channel = uuid.uuid4().hex
@@ -549,7 +549,7 @@ async def test_when_not_renewable(self):
 
         self.ably = await TestApp.get_ably_rest(
             key=None,
-            rest_host=self.host,
+            endpoint=self.host,
             token='token ID cannot be used to create a new token',
             use_binary_protocol=False)
         await self.ably.channels[self.channel].publish('evt', 'msg')
@@ -568,7 +568,7 @@ async def test_when_not_renewable_with_token_details(self):
         token_details = TokenDetails(token='a_dummy_token')
         self.ably = await TestApp.get_ably_rest(
             key=None,
-            rest_host=self.host,
+            endpoint=self.host,
             token_details=token_details,
             use_binary_protocol=False)
         await self.ably.channels[self.channel].publish('evt', 'msg')
@@ -638,7 +638,7 @@ def cb_publish(request):
 
     # RSA4b1
     async def test_query_time_false(self):
-        ably = await TestApp.get_ably_rest(rest_host=self.host)
+        ably = await TestApp.get_ably_rest(endpoint=self.host)
         await ably.auth.authorize()
         self.publish_fail = True
         await ably.channels[self.channel].publish('evt', 'msg')
@@ -647,7 +647,7 @@ async def test_query_time_false(self):
 
     # RSA4b1
     async def test_query_time_true(self):
-        ably = await TestApp.get_ably_rest(query_time=True, rest_host=self.host)
+        ably = await TestApp.get_ably_rest(query_time=True, endpoint=self.host)
         await ably.auth.authorize()
         self.publish_fail = False
         await ably.channels[self.channel].publish('evt', 'msg')