From ca4f4a3222f5c5dd090aba65b3a9e199dfe7cb29 Mon Sep 17 00:00:00 2001 From: Daniel Lenski Date: Thu, 30 Dec 2021 15:53:51 -0500 Subject: [PATCH 1/3] Use identical download sizes for HTTP- and socket-based download tests This requires switching the HTTP download test from the `/speedtest/randomNxN.jpg` endpoint to the `/download?size=N` endpoint, as used by the web interface of Speedtest.net currently. --- speedtest.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/speedtest.py b/speedtest.py index 0110718be..fb3fd3c23 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1219,13 +1219,9 @@ def get_config(self): sizes = { 'upload': up_sizes[ratio - 1:], } - if self._use_socket: - sizes['download'] = [245388, 505544, 1118012, 1986284, 4468241, - 7907740, 12407926, 17816816, 24262167, - 31625365] - else: - sizes['download'] = [350, 500, 750, 1000, 1500, 2000, 2500, - 3000, 3500, 4000] + sizes['download'] = [245388, 505544, 1118012, 1986284, 4468241, + 7907740, 12407926, 17816816, 24262167, + 31625365] size_count = len(sizes['upload']) @@ -1703,8 +1699,8 @@ def download(self, callback=do_nothing): for size in self.config['sizes']['download']: for _ in range(0, self.config['counts']['download']): urls.append( - '%s/random%sx%s.jpg' % - (os.path.dirname(self.best['url']), size, size) + '%s/download?size=%d' % + (os.path.dirname(os.path.dirname(self.best['url'])), size) ) request_count = len(urls) From 0096adccf70bbec464c0b3c60c85afc88d51a08e Mon Sep 17 00:00:00 2001 From: Daniel Lenski Date: Thu, 30 Dec 2021 16:03:50 -0500 Subject: [PATCH 2/3] Support JSON server list in addition to XML server list(s) The JSON server list (https://www.speedtest.net/api/js/servers) is what the Speedtest.net web interface currently uses. --- speedtest.py | 66 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/speedtest.py b/speedtest.py index fb3fd3c23..92acb8517 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1286,6 +1286,7 @@ def get_servers(self, servers=None, exclude=None): ) urls = [ + 'https://www.speedtest.net/api/js/servers', '://www.speedtest.net/speedtest-servers-static.php', 'http://c.speedtest.net/speedtest-servers-static.php', '://www.speedtest.net/speedtest-servers.php', @@ -1311,6 +1312,10 @@ def get_servers(self, servers=None, exclude=None): raise ServersRetrievalError() stream = get_response_stream(uh) + try: + is_json = uh.headers.getheader('content-type').startswith('application/json') + except AttributeError: + is_json = uh.getheader('content-type').startswith('application/json') serversxml_list = [] while 1: @@ -1328,37 +1333,48 @@ def get_servers(self, servers=None, exclude=None): raise ServersRetrievalError() serversxml = ''.encode().join(serversxml_list) + attriblist = [] - printer('Servers XML:\n%s' % serversxml, debug=True) + if is_json: + printer('Servers JSON:\n%s' % serversxml, debug=True) + try: + attriblist = json.loads(serversxml) + except (ValueError, json.JSONDecodeError): + raise ServersRetrievalError() + + else: + printer('Servers XML:\n%s' % serversxml, debug=True) - try: try: try: - root = ET.fromstring(serversxml) - except ET.ParseError: - e = get_exception() - raise SpeedtestServersError( - 'Malformed speedtest.net server list: %s' % e - ) - elements = root.getiterator('server') - except AttributeError: + try: + root = ET.fromstring(serversxml) + except ET.ParseError: + e = get_exception() + raise SpeedtestServersError( + 'Malformed speedtest.net server list: %s' % e + ) + elements = root.getiterator('server') + except AttributeError: + try: + root = DOM.parseString(serversxml) + except ExpatError: + e = get_exception() + raise SpeedtestServersError( + 'Malformed speedtest.net server list: %s' % e + ) + elements = root.getElementsByTagName('server') + except (SyntaxError, xml.parsers.expat.ExpatError): + raise ServersRetrievalError() + + for server in elements: try: - root = DOM.parseString(serversxml) - except ExpatError: - e = get_exception() - raise SpeedtestServersError( - 'Malformed speedtest.net server list: %s' % e - ) - elements = root.getElementsByTagName('server') - except (SyntaxError, xml.parsers.expat.ExpatError): - raise ServersRetrievalError() - - for server in elements: - try: - attrib = server.attrib - except AttributeError: - attrib = dict(list(server.attributes.items())) + attrib = server.attrib + except AttributeError: + attrib = dict(list(server.attributes.items())) + attriblist.append(attrib) + for attrib in attriblist: if servers and int(attrib.get('id')) not in servers: continue From 6c6fb03040eacb108aafd715f9e156f7b5bf66d2 Mon Sep 17 00:00:00 2001 From: Daniel Lenski Date: Thu, 30 Dec 2021 16:04:15 -0500 Subject: [PATCH 3/3] Comments, code simplification --- speedtest.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/speedtest.py b/speedtest.py index 92acb8517..5bcadaa0e 100755 --- a/speedtest.py +++ b/speedtest.py @@ -1376,28 +1376,25 @@ def get_servers(self, servers=None, exclude=None): for attrib in attriblist: if servers and int(attrib.get('id')) not in servers: + # Not one of the preselected servers continue if (int(attrib.get('id')) in self.config['ignore_servers'] or int(attrib.get('id')) in exclude): + # One of the specifically excluded or ignored servers continue host, port = attrib['host'].split(':') attrib['host'] = (host, int(port)) try: - d = distance(self.lat_lon, - (float(attrib.get('lat')), - float(attrib.get('lon')))) + d = attrib['d'] = distance(self.lat_lon, + (float(attrib.get('lat')), + float(attrib.get('lon')))) except Exception: continue - attrib['d'] = d - - try: - self.servers[d].append(attrib) - except KeyError: - self.servers[d] = [attrib] + self.servers.setdefault(d, []).append(attrib) break