Add beginning of larger tutorial example.

Author: pya

examples/aiohttp_client.py

@@ -0,0 +1,50 @@
+"""aiohttp-based client to retrieve web pages.
+"""
+
+import asyncio
+from contextlib import closing
+import time
+
+import aiohttp
+
+
+async def fetch_page(session, host, port=8000, wait=0):
+    """Get one page.
+    """
+    url = '{}:{}/{}'.format(host, port, wait)
+    with aiohttp.Timeout(10):
+        async with session.get(url) as response:
+            assert response.status == 200
+            return await response.text()
+
+
+def get_multiple_pages(host, waits, port=8000, show_time=True):
+    """Get multiple pages.
+    """
+    tasks = []
+    pages = []
+    start = time.perf_counter()
+    with closing(asyncio.get_event_loop()) as loop:
+        with aiohttp.ClientSession(loop=loop) as session:
+            for wait in waits:
+                tasks.append(fetch_page(session, host, port, wait))
+            pages = loop.run_until_complete(asyncio.gather(*tasks))
+    duration = time.perf_counter() - start
+    sum_waits = sum(waits)
+    if show_time:
+        msg = 'It took {:4.2f} seconds for a total waiting time of {:4.2f}.'
+        print(msg.format(duration, sum_waits))
+    return pages
+
+
+if __name__ == '__main__':
+
+    def main():
+        """Test it.
+        """
+        pages = get_multiple_pages(host='http://localhost', port='8000',
+                                   waits=[1, 5, 3, 2])
+        for page in pages:
+            print(page)
+
+    main()

examples/async_client_blocking.py

@@ -0,0 +1,39 @@
+"""Get "web pages.
+
+Waiting until one pages is download before getting the next."
+"""
+
+import asyncio
+from contextlib import closing
+import time
+
+from async_page import get_page
+
+
+def get_multiple_pages(host, port, waits, show_time=True):
+    """Get multiple pages.
+    """
+    start = time.perf_counter()
+    pages = []
+    with closing(asyncio.get_event_loop()) as loop:
+        for wait in waits:
+            pages.append(loop.run_until_complete(get_page(host, port, wait)))
+
+    duration = time.perf_counter() - start
+    sum_waits = sum(waits)
+    if show_time:
+        msg = 'It took {:4.2f} seconds for a total waiting time of {:4.2f}.'
+        print(msg.format(duration, sum_waits))
+    return pages
+
+if __name__ == '__main__':
+
+    def main():
+        """Test it.
+        """
+        pages = get_multiple_pages(host='localhost', port='8000',
+                                   waits=[1, 5, 3, 2])
+        for page in pages:
+            print(page)
+
+    main()

examples/async_client_nonblocking.py

@@ -0,0 +1,40 @@
+"""Get "web pages.
+
+Waiting until one pages is download before getting the next."
+"""
+
+import asyncio
+from contextlib import closing
+import time
+
+from async_page import get_page
+
+
+def get_multiple_pages(host, port, waits, show_time=True):
+    """Get multiple pages.
+    """
+    start = time.perf_counter()
+    pages = []
+    tasks = []
+    with closing(asyncio.get_event_loop()) as loop:
+        for wait in waits:
+            tasks.append(get_page(host, port, wait))
+        pages = loop.run_until_complete(asyncio.gather(*tasks))
+    duration = time.perf_counter() - start
+    sum_waits = sum(waits)
+    if show_time:
+        msg = 'It took {:4.2f} seconds for a total waiting time of {:4.2f}.'
+        print(msg.format(duration, sum_waits))
+    return pages
+
+if __name__ == '__main__':
+
+    def main():
+        """Test it.
+        """
+        pages = get_multiple_pages(host='localhost', port='8000',
+                                   waits=[1, 5, 3, 2])
+        for page in pages:
+            print(page)
+
+    main()

examples/async_page.py

@@ -0,0 +1,42 @@
+"""Get a "web page" asynchronously.
+"""
+
+import asyncio
+
+ENCODING = 'ISO-8859-1'
+
+
+def get_encoding(header):
+    """Find out encoding.
+    """
+    for line in header:
+        if line.lstrip().startswith('Content-type'):
+            for entry in line.split(';'):
+                if entry.strip().startswith('charset'):
+                    return entry.split('=')[1].strip()
+    return ENCODING
+
+
+async def get_page(host, port, wait=0):
+    """Get a "web page" asynchronously.
+    """
+    reader, writer = await asyncio.open_connection(host, port)
+    writer.write(b'\r\n'.join([
+        'GET /{} HTTP/1.0'.format(wait).encode(ENCODING),
+        b'Host: %b' % host.encode(ENCODING),
+        b'Connection: close',
+        b'', b''
+    ]))
+    msg_lines = []
+    header = []
+    async for raw_line in reader:
+        line = raw_line.decode(ENCODING).strip()
+        if not line.strip():
+            break
+        header.append(line)
+    encoding = get_encoding(header)
+    async for raw_line in reader:
+        line = raw_line.decode(encoding).strip()
+        msg_lines.append(line)
+    writer.close()
+    return '\n'.join(msg_lines)

examples/simple_server.py

@@ -0,0 +1,53 @@
+# file: simple_server.py
+
+"""Simple HTTP server with GET that waits for given seconds.
+"""
+
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from socketserver import ThreadingMixIn
+import time
+
+
+ENCODING = 'utf-8'
+
+
+class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
+    """Simple multi-threaded HTTP server.
+    """
+    pass
+
+
+class MyRequestHandler(BaseHTTPRequestHandler):
+    """Very simple request handler. Only supports GET.
+    """
+
+    def do_GET(self):  # pylint: disable=invalid-name
+        """Respond after seconds given in path.
+        """
+        try:
+            seconds = float(self.path[1:])
+        except ValueError:
+            seconds = 0.0
+        text = "Waited for {:4.2f} seconds.\nThat's all.\n"
+        msg = text.format(seconds).encode(ENCODING)
+        time.sleep(seconds)
+        self.send_response(200)
+        self.send_header("Content-type", 'text/plain; charset=utf-8')
+        self.send_header("Content-length", str(len(msg)))
+        self.end_headers()
+        self.wfile.write(msg)
+
+
+def run(server_class=ThreadingHTTPServer,
+        handler_class=MyRequestHandler,
+        port=8000):
+    """Run the simple server on given port.
+    """
+    server_address = ('', port)
+    httpd = server_class(server_address, handler_class)
+    print('Serving from port {} ...'.format(port))
+    httpd.serve_forever()
+
+
+if __name__ == '__main__':
+    run()

examples/synchronous_client.py

@@ -0,0 +1,55 @@
+"""Synchronous client to retrieve web pages.
+"""
+
+
+from urllib.request import urlopen
+import time
+
+ENCODING = 'ISO-8859-1'
+
+
+def get_encoding(http_response):
+    """Find out encoding.
+    """
+    content_type = http_response.getheader('Content-type')
+    for entry in content_type.split(';'):
+        if entry.strip().startswith('charset'):
+            return entry.split('=')[1].strip()
+    return ENCODING
+
+
+def get_page(host, port, wait=0):
+    """Get one page suppling `wait` time.
+
+    The path will be build with: `host:port/wait`
+    """
+    full_url = '{}:{}/{}'.format(host, port, wait)
+    with urlopen(full_url) as http_response:
+        html = http_response.read().decode(get_encoding(http_response))
+    return html
+
+
+def get_multiple_pages(host, port, waits, show_time=True):
+    """Get multiple pages.
+    """
+    start = time.perf_counter()
+    pages = [get_page(host, port, wait) for wait in waits]
+    duration = time.perf_counter() - start
+    sum_waits = sum(waits)
+    if show_time:
+        msg = 'It took {:4.2f} seconds for a total waiting time of {:4.2f}.'
+        print(msg.format(duration, sum_waits))
+    return pages
+
+
+if __name__ == '__main__':
+
+    def main():
+        """Test it.
+        """
+        pages = get_multiple_pages(host='http://localhost', port='8000',
+                                   waits=[1, 5, 3, 2])
+        for page in pages:
+            print(page)
+
+    main()

index.rst

@@ -13,6 +13,7 @@ Chapter 1: First steps with asyncio
    hello_clock.rst
    http_client.rst
    twisted.rst
+   webscraper.rst
    getting_help.rst