From 231c702715ab89e6e7be3d9d2211bb1d04d8befa Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 25 Jun 2025 16:37:22 +0100 Subject: [PATCH] httpx / ahttpx switcher --- docs/connections.md | 138 +++++++++++++++++++++++++++++----- docs/content-types.md | 138 +++++++++++++++++++--------------- docs/networking.md | 166 ++++++++++++++++++++++++++++++++++++++--- src/ahttpx/_content.py | 12 +++ src/httpx/_content.py | 12 +++ src/httpx/_server.py | 2 +- 6 files changed, 378 insertions(+), 90 deletions(-) diff --git a/docs/connections.md b/docs/connections.md index a442439..5698bb7 100644 --- a/docs/connections.md +++ b/docs/connections.md @@ -4,7 +4,9 @@ The mechanics of sending HTTP requests is dealt with by the `ConnectionPool` and We can introspect a `Client` instance to get some visibility onto the state of the connection pool. -```python +
httpx
+ +```{ .python .httpx } >>> with httpx.Client() as cli >>> urls = [ ... "https://www.wikipedia.org/", @@ -23,6 +25,25 @@ We can introspect a `Client` instance to get some visibility onto the state of t ... # ] ``` +```{ .python .ahttpx .hidden } +>>> async with ahttpx.Client() as cli +>>> urls = [ +... "https://www.wikipedia.org/", +... "https://www.theguardian.com/", +... "https://news.ycombinator.com/", +... ] +... for url in urls: +... await cli.get(url) +... print(cli.transport) +... # +... print(cli.transport.connections) +... # [ +... # , +... # , +... # , +... # ] +``` + --- ## Understanding the stack @@ -31,25 +52,46 @@ The `Client` class is responsible for handling redirects and cookies. It also ensures that outgoing requests include a default set of headers such as `User-Agent` and `Accept-Encoding`. -```python -with httpx.Client() as cli: - r = cli.request("GET", "https://www.example.com/") +
httpx
+ +```{ .python .httpx } +>>> with httpx.Client() as cli: +>>> r = cli.request("GET", "https://www.example.com/") +``` + +```{ .python .ahttpx .hidden } +>>> async with ahttpx.Client() as cli: +>>> r = await cli.request("GET", "https://www.example.com/") ``` The `Client` class sends requests using a `ConnectionPool`, which is responsible for managing a pool of HTTP connections. This ensures quicker and more efficient use of resources than opening and closing a TCP connection with each request. The connection pool also handles HTTP proxying if required. A single connection pool is able to handle multiple concurrent requests, with locking in place to ensure that the pool does not become over-saturated. -```python -with httpx.ConnectionPool() as pool: - r = pool.request("GET", "https://www.example.com/") +
httpx
+ +```{ .python .httpx } +>>> with httpx.ConnectionPool() as pool: +>>> r = pool.request("GET", "https://www.example.com/") +``` + +```{ .python .ahttpx .hidden } +>>> async with ahttpx.ConnectionPool() as pool: +>>> r = await pool.request("GET", "https://www.example.com/") ``` Individual HTTP connections can be managed directly with the `Connection` class. A single connection can only handle requests sequentially. Locking is provided to ensure that requests are strictly queued sequentially. -```python -with httpx.open_connection("https://www.example.com/") as conn: - r = conn.request("GET", "/") +
httpx
+ +```{ .python .httpx } +>>> with httpx.open_connection("https://www.example.com/") as conn: +>>> r = conn.request("GET", "/") +``` + +```{ .python .ahttpx .hidden } +>>> async with ahttpx.open_connection("https://www.example.com/") as conn: +>>> r = await conn.request("GET", "/") ``` Protocol handling is dealt with using [the `h11` package](https://h11.readthedocs.io/en/latest/), a rigorously designed HTTP/1.1 implementation which follows [the Sans-IO design pattern](https://sans-io.readthedocs.io/). @@ -60,33 +102,61 @@ The `NetworkBackend` is responsible for managing the TCP stream, providing a raw ## ConnectionPool -```python +
httpx
+ +```{ .python .httpx } >>> pool = httpx.ConnectionPool() >>> pool ``` +```{ .python .ahttpx .hidden } +>>> pool = ahttpx.ConnectionPool() +>>> pool + +``` + ### `.request(method, url, headers=None, content=None)` -```python +
httpx
+ +```{ .python .httpx } >>> with httpx.ConnectionPool() as pool: >>> res = pool.request("GET", "https://www.example.com") >>> res, pool , ``` +```{ .python .ahttpx .hidden } +>>> async with ahttpx.ConnectionPool() as pool: +>>> res = await pool.request("GET", "https://www.example.com") +>>> res, pool +, +``` + ### `.stream(method, url, headers=None, content=None)` -```python +
httpx
+ +```{ .python .httpx } >>> with httpx.ConnectionPool() as pool: >>> with pool.stream("GET", "https://www.example.com") as res: >>> res, pool , ``` +```{ .python .ahttpx .hidden } +>>> with ahttpx.ConnectionPool() as pool: +>>> with await pool.stream("GET", "https://www.example.com") as res: +>>> res, pool +, +``` + ### `.send(request)` -```python +
httpx
+ +```{ .python .httpx } >>> with httpx.ConnectionPool() as pool: >>> req = httpx.Request("GET", "https://www.example.com") >>> with pool.send(req) as res: @@ -95,14 +165,31 @@ The `NetworkBackend` is responsible for managing the TCP stream, providing a raw , ``` +```{ .python .ahttpx .hidden } +>>> with ahttpx.ConnectionPool() as pool: +>>> req = ahttpx.Request("GET", "https://www.example.com") +>>> async with await pool.send(req) as res: +>>> await res.read() +>>> res, pool +, +``` + ### `.close()` -```python +
httpx
+ +```{ .python .httpx } >>> with httpx.ConnectionPool() as pool: >>> pool.close() ``` +```{ .python .ahttpx .hidden } +>>> with ahttpx.ConnectionPool() as pool: +>>> await pool.close() + +``` + --- ## Connection @@ -113,23 +200,40 @@ The `NetworkBackend` is responsible for managing the TCP stream, providing a raw ## Protocol upgrades -```python +
httpx
+ +```{ .python .httpx } with httpx.open_connection("https://www.example.com/") as conn: with conn.upgrade("GET", "/feed", {"Upgrade": "WebSocket"}) as stream: ... ``` +```{ .python .ahttpx .hidden } +async with await ahttpx.open_connection("https://www.example.com/") as conn: + async with await conn.upgrade("GET", "/feed", {"Upgrade": "WebSocket"}) as stream: + ... +``` + `` ## Proxy `CONNECT` requests -```python +
httpx
+ +```{ .python .httpx } with httpx.open_connection("http://127.0.0.1:8080") as conn: with conn.upgrade("CONNECT", "www.encode.io:443") as stream: stream.start_tls(ctx, hostname="www.encode.io") ... ``` +```{ .python .ahttpx .hidden } +async with await ahttpx.open_connection("http://127.0.0.1:8080") as conn: + async with await conn.upgrade("CONNECT", "www.encode.io:443") as stream: + await stream.start_tls(ctx, hostname="www.encode.io") + ... +``` + `` --- diff --git a/docs/content-types.md b/docs/content-types.md index 3c529dd..557ed63 100644 --- a/docs/content-types.md +++ b/docs/content-types.md @@ -10,12 +10,20 @@ The most common content types for upload data are... Content can be included directly in a request by using bytes or a byte iterator and setting the appropriate `Content-Type` header. -```python +
httpx
+ +```{ .python .httpx } >>> headers = {'Content-Type': 'application/json'} ->>> content = json.dumps({{"number": 123.5, "bool": [True, False], "text": "hello"}) +>>> content = json.dumps({"number": 123.5, "bool": [True, False], "text": "hello"}) >>> response = cli.put(url, headers=headers, content=content) ``` +```{ .python .ahttpx .hidden } +>>> headers = {'Content-Type': 'application/json'} +>>> content = json.dumps({"number": 123.5, "bool": [True, False], "text": "hello"}) +>>> response = await cli.put(url, headers=headers, content=content) +``` + There are also several classes provided for setting the request content. These implement either the `Content` or `StreamingContent` API, and handle constructing the content and setting the relevant headers. * `
` @@ -26,9 +34,16 @@ There are also several classes provided for setting the request content. These i For example, sending a JSON request... -```python ->>> data = {"number": 123.5, "bool": [True, False], "text": "hello"} ->>> cli.post(url, content=httpx.JSON(data)) +
httpx
+ +```{ .python .httpx } +>>> data = httpx.JSON({"number": 123.5, "bool": [True, False], "text": "hello"}) +>>> cli.post(url, content=data) +``` + +```{ .python .ahttpx .hidden } +>>> data = httpx.JSON({"number": 123.5, "bool": [True, False], "text": "hello"}) +>>> await cli.post(url, content=data) ``` --- @@ -37,7 +52,9 @@ For example, sending a JSON request... The `Form` class provides an immutable multi-dict for accessing HTML form data. This class implements the `Content` interface, allowing for HTML form uploads. -```python +
httpx
+ +```{ .python .httpx } >>> form = httpx.Form({'name': '...'}) >>> form ... @@ -47,11 +64,23 @@ The `Form` class provides an immutable multi-dict for accessing HTML form data. ... ``` +```{ .python .ahttpx .hidden } +>>> form = httpx.Form({'name': '...'}) +>>> form +... +>>> form['name'] +... +>>> res = await cli.post(url, content=form) +... +``` + ## Files The `Files` class provides an immutable multi-dict for accessing HTML form file uploads. This class implements the `StreamingContent` interface, allowing for HTML form file uploads. -```python +
httpx
+ +```{ .python .httpx } >>> files = httpx.Files({'upload': httpx.File('data.json')}) >>> files ... @@ -61,11 +90,23 @@ The `Files` class provides an immutable multi-dict for accessing HTML form file ... ``` +```{ .python .ahttpx .hidden } +>>> files = httpx.Files({'upload': httpx.File('data.json')}) +>>> files +... +>>> files['upload'] +... +>>> res = await cli.post(url, content=files) +... +``` + ## MultiPart The `MultiPart` class provides a wrapper for HTML form and files uploads. This class implements the `StreamingContent` interface, allowing for allowing for HTML form uploads including both data and file uploads. -```python +
httpx
+ +```{ .python .httpx } >>> multipart = httpx.MultiPart(form={'name': '...'}, files={'avatar': httpx.File('image.png')}) >>> multipart.form['name'] ... @@ -74,57 +115,46 @@ The `MultiPart` class provides a wrapper for HTML form and files uploads. This c >>> res = cli.post(url, content=multipart) ``` -## File - -The `File` class provides a wrapper for file uploads, and is used for uploads instead of passing a file object directly. File uploads can be from any data source, although they will most typically be from a file on the file system. - -```python -File('upload.json') +```{ .python .ahttpx .hidden } +>>> multipart = httpx.MultiPart(form={'name': '...'}, files={'avatar': httpx.File('image.png')}) +>>> multipart.form['name'] +... +>>> multipart.files['avatar'] +... +>>> res = await cli.post(url, content=multipart) ``` -* `source` - The data source for the upload. For file system sources, this may be a path as a string or `pathlib.Path` instance. For other sources this should be a bytes iterator. -* `filename` - A string representing the filename for the upload. If the `source` is provided as a string or `pathlib.Path` instance then this is set automatically, otherwise it is a required argument. -* `size` - Either `None`, or an int representing the size of the file upload. If `source` is provided as a string or `pathlib.Path` instance, then the file size will be automatically determined. -* `content_type` - Either `None`, or a string representing the media type of the file, as used in the `Content-Type` header. If `source` is provided as a string or `pathlib.Path` instance, then a content type based on the file extension will be automatically determined. +## File + +The `File` class provides a wrapper for file uploads, and is used for uploads instead of passing a file object directly. +
httpx
-```python -# Use a streaming download ... -headers = {'Accept-Encoding': 'identity'} -with cli.stream('...', headers=headers) as response: - source = response.stream - filename = 'upload.json' - size = int(response.headers['Content-Length']) if 'Content-Length' in response.headers else None - upload = File(source, filename='upload.json', size=size) +```{ .python .httpx } +>>> file = httpx.File('upload.json') +>>> cli.post(url, content=file) +``` - # Upload the ... - cli.put('https://...', Files({'upload': upload})) +```{ .python .ahttpx .hidden } +>>> file = httpx.File('upload.json') +>>> await cli.post(url, content=file) ``` ## JSON The `JSON` class provides a wrapper for JSON uploads. This class implements the `Content` interface, allowing for HTTP JSON uploads. -```python ->>> cli.put(url, content=httpx.JSON(data)) -``` - -Is equivelent to... +
httpx
-```python ->>> options = {'allow_nan': False, 'ensure_ascii': False, 'separators': (",", ":")} ->>> headers = {'Content-Type': 'application/json'} ->>> content = json.dumps(data, **options).encode('utf-8') ->>> cli.put(url, headers=headers, content=content) +```{ .python .httpx } +>>> data = httpx.JSON({...}) +>>> cli.put(url, content=data) ``` -https://www.rfc-editor.org/rfc/rfc8259#section-8.1 - -* `allow_nan = False` - ... -* `ensure_ascii = False` - ... -* `separators=(",", ":")` - ... - -For an alternative... https://github.com/ijl/orjson +```{ .python .ahttpx .hidden } +>>> data = httpx.JSON({...}) +>>> await cli.put(url, content=data) +``` --- @@ -132,23 +162,9 @@ For an alternative... https://github.com/ijl/orjson An interface for constructing HTTP content, along with relevant headers. -The following methods must be implemented... - -* `.get_headers()` - Returns an `httx.Headers` instance. -* `.get_content()` - Returns bytes used to set the body of the request. - -Implmentations should typically set a `Content-Type` header indicating the correct media type. - -## StreamingContent - -An interface for constructing streaming HTTP content, along with relevant headers. - -The following methods must be implemented... - -* `.get_headers()` - Returns an `httx.Headers` instance. -* `.get_stream()` - Returns a bytes iterator used to set the body of the request. +The following method must be implemented... -Implmentations should typically set a `Content-Type` header indicating the correct media type. For some streaming content the upload size can be determined in advance, in these cases a `Content-Length` header should also be included. +* `.encode()` - Returns a tuple of `(httx.Stream, str)`, representing the encoded data and the content type. --- diff --git a/docs/networking.md b/docs/networking.md index 0be863d..efd0518 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -8,26 +8,45 @@ The lowest level network abstractions in `httpx` are the `NetworkBackend` and `N The default backend is instantiated via the `NetworkBackend` class... -```python +
httpx
+ +```{ .python .httpx } >>> net = httpx.NetworkBackend() >>> net ``` +```{ .python .ahttpx .hidden } +>>> net = ahttpx.NetworkBackend() +>>> net + +``` + ### `.connect(host, port)` A TCP stream is created using the `connect` method... -```python +
httpx
+ +```{ .python .httpx } >>> net = httpx.NetworkBackend() >>> stream = net.connect("www.encode.io", 80) >>> stream ``` +```{ .python .ahttpx .hidden } +>>> net = ahttpx.NetworkBackend() +>>> stream = await net.connect("www.encode.io", 80) +>>> stream + +``` + Streams support being used in a context managed style. The cleanest approach to resource management is to use `.connect(...)` in the context of a `with` block. -```python +
httpx
+ +```{ .python .httpx } >>> net = httpx.NetworkBackend() >>> with net.connect("dev.encode.io", 80) as stream: >>> ... @@ -35,6 +54,14 @@ Streams support being used in a context managed style. The cleanest approach to ``` +```{ .python .ahttpx .hidden } +>>> net = ahttpx.NetworkBackend() +>>> async with await net.connect("dev.encode.io", 80) as stream: +>>> ... +>>> stream + +``` + ## `NetworkStream(sock)` The `NetworkStream` class provides TCP stream abstraction, by providing a thin wrapper around a socket instance. @@ -70,7 +97,9 @@ Return information about the underlying resource. May include... Close the network stream. For TLS streams this will attempt to send a closing handshake before terminating the conmection. -```python +
httpx
+ +```{ .python .httpx } >>> net = httpx.NetworkBackend() >>> stream = net.connect("dev.encode.io", 80) >>> try: @@ -81,6 +110,17 @@ Close the network stream. For TLS streams this will attempt to send a closing ha ``` +```{ .python .ahttpx .hidden } +>>> net = ahttpx.NetworkBackend() +>>> stream = await net.connect("dev.encode.io", 80) +>>> try: +>>> ... +>>> finally: +>>> await stream.close() +>>> stream + +``` + --- ## Timeouts @@ -95,20 +135,31 @@ The timeout context manager can be used to wrap socket operations anywhere in th Here's an example of enforcing an overall 3 second timeout on a request. -```python +
httpx
+ +```{ .python .httpx } >>> with httpx.Client() as cli: >>> with httpx.timeout(3.0): >>> res = cli.get('https://www.example.com') >>> print(res) ``` +```{ .python .ahttpx .hidden } +>>> async with ahttpx.Client() as cli: +>>> async with ahttpx.timeout(3.0): +>>> res = await cli.get('https://www.example.com') +>>> print(res) +``` + Timeout contexts provide an API allowing for deadlines to be cancelled. ### .cancel() In this example we enforce a 3 second timeout on *receiving the start of* a streaming HTTP response... -```python +
httpx
+ +```{ .python .httpx } >>> with httpx.Client() as cli: >>> with httpx.timeout(3.0) as t: >>> with cli.stream('https://www.example.com') as r: @@ -118,13 +169,25 @@ In this example we enforce a 3 second timeout on *receiving the start of* a stre >>> print("...", chunk) ``` +```{ .python .ahttpx .hidden } +>>> async with ahttpx.Client() as cli: +>>> async with ahttpx.timeout(3.0) as t: +>>> async with await cli.stream('https://www.example.com') as r: +>>> t.cancel() +>>> print(">>>", res) +>>> async for chunk in r.stream: +>>> print("...", chunk) +``` + --- ## Sending HTTP requests Let's take a look at how we can work directly with a network backend to send an HTTP request, and recieve an HTTP response. -```python +
httpx
+ +```{ .python .httpx } import httpx import ssl import truststore @@ -156,6 +219,38 @@ with httpx.timeout(10.0): resp = b''.join(buffer) ``` +```{ .python .ahttpx .hidden } +import ahttpx +import ssl +import truststore + +net = ahttpx.NetworkBackend() +ctx = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) +req = b'\r\n'.join([ + b'GET / HTTP/1.1', + b'Host: www.example.com', + b'User-Agent: python/dev', + b'Connection: close', + b'', + b'', +]) + +# Use a 10 second overall timeout for the entire request/response. +async with ahttpx.timeout(10.0): + # Use a 3 second timeout for the initial connection. + async with ahttpx.timeout(3.0) as t: + # Open the connection & establish SSL. + async with await net.connect("www.example.com", 443) as stream: + await stream.start_tls(ctx, hostname="www.example.com") + t.cancel() + # Send the request & read the response. + await stream.write(req) + buffer = [] + while part := await stream.read(): + buffer.append(part) + resp = b''.join(buffer) +``` + The example above is somewhat contrived, there's no HTTP parsing implemented so we can't actually determine when the response is complete. We're using a `Connection: close` header to request that the server close the connection once the response is complete. A more complete example would require proper HTTP parsing. The `Connection` class implements an HTTP request/response interface, layered over a `NetworkStream`. @@ -178,8 +273,10 @@ The abstract interface implemented by `NetworkStream`. See above for details. We can use these interfaces to implement custom functionality. For example, here we're providing a network backend that logs all the ingoing and outgoing bytes. -```python -class RecordingBackend(NetworkBackendInterface): +
httpx
+ +```{ .python .httpx } +class RecordingBackend(httpx.NetworkBackendInterface): def __init__(self): self._backend = NetworkBackend() @@ -190,7 +287,7 @@ class RecordingBackend(NetworkBackendInterface): return RecordingStream(stream) -class RecordingStream(NetworkStreamInterface): +class RecordingStream(httpx.NetworkStreamInterface): def __init__(self, stream): self._stream = stream @@ -219,14 +316,61 @@ class RecordingStream(NetworkStreamInterface): self._stream.close() ``` +```{ .python .ahttpx .hidden } +class RecordingBackend(ahhtpx.NetworkBackendInterface): + def __init__(self): + self._backend = NetworkBackend() + + async def connect(self, host, port): + # Delegate creating connections to the default + # network backend, and return a wrapped stream. + stream = await self._backend.connect(host, port) + return RecordingStream(stream) + + +class RecordingStream(ahttpx.NetworkStreamInterface): + def __init__(self, stream): + self._stream = stream + + async def read(self, max_bytes: int = None): + # Print all incoming data to the terminal. + data = await self._stream.read(max_bytes) + lines = data.decode('ascii', errors='replace').splitlines() + for line in lines: + print("<<< ", line) + return data + + async def write(self, data): + # Print all outgoing data to the terminal. + lines = data.decode('ascii', errors='replace').splitlines() + for line in lines: + print(">>> ", line) + await self._stream.write(data) + + async def start_tls(ctx, hostname): + await self._stream.start_tls(ctx, hostname) + + def get_extra_info(key): + return self._stream.get_extra_info(key) + + async def close(): + await self._stream.close() +``` + We can now instantiate a client using this network backend. -```python +```{ .python .httpx } >>> transport = httpx.ConnectionPool(backend=RecordingBackend()) >>> cli = httpx.Client(transport=transport) >>> cli.get('https://www.example.com') ``` +```{ .python .ahttpx .hidden } +>>> transport = ahttpx.ConnectionPool(backend=RecordingBackend()) +>>> cli = ahttpx.Client(transport=transport) +>>> await cli.get('https://www.example.com') +``` + Custom network backends can also be used to provide functionality such as handling DNS caching for name lookups, or connecting via a UNIX domain socket instead of a TCP connection. --- diff --git a/src/ahttpx/_content.py b/src/ahttpx/_content.py index e75d30f..e8afdb8 100644 --- a/src/ahttpx/_content.py +++ b/src/ahttpx/_content.py @@ -293,6 +293,9 @@ def encode(self) -> tuple[Stream, str]: content_type = "application/json" return (stream, content_type) + def __repr__(self) -> str: + return f"" + class Text(Content): def __init__(self, text: str) -> None: @@ -303,6 +306,9 @@ def encode(self) -> tuple[Stream, str]: content_type = "text/plain; charset='utf-8'" return (stream, content_type) + def __repr__(self) -> str: + return f"" + class HTML(Content): def __init__(self, text: str) -> None: @@ -313,6 +319,9 @@ def encode(self) -> tuple[Stream, str]: content_type = "text/html; charset='utf-8'" return (stream, content_type) + def __repr__(self) -> str: + return f"" + class MultiPart(Content): def __init__( @@ -375,3 +384,6 @@ async def iter_bytes(self) -> typing.AsyncIterator[bytes]: yield "\r\n".encode("utf-8") yield f"--{self._boundary}--\r\n".encode("utf-8") + + def __repr__(self) -> str: + return f"" diff --git a/src/httpx/_content.py b/src/httpx/_content.py index 1ee0ef2..f430590 100644 --- a/src/httpx/_content.py +++ b/src/httpx/_content.py @@ -293,6 +293,9 @@ def encode(self) -> tuple[Stream, str]: content_type = "application/json" return (stream, content_type) + def __repr__(self) -> str: + return f"" + class Text(Content): def __init__(self, text: str) -> None: @@ -303,6 +306,9 @@ def encode(self) -> tuple[Stream, str]: content_type = "text/plain; charset='utf-8'" return (stream, content_type) + def __repr__(self) -> str: + return f"" + class HTML(Content): def __init__(self, text: str) -> None: @@ -313,6 +319,9 @@ def encode(self) -> tuple[Stream, str]: content_type = "text/html; charset='utf-8'" return (stream, content_type) + def __repr__(self) -> str: + return f"" + class MultiPart(Content): def __init__( @@ -375,3 +384,6 @@ def iter_bytes(self) -> typing.Iterator[bytes]: yield "\r\n".encode("utf-8") yield f"--{self._boundary}--\r\n".encode("utf-8") + + def __repr__(self) -> str: + return f"" diff --git a/src/httpx/_server.py b/src/httpx/_server.py index 6f124d4..27021de 100644 --- a/src/httpx/_server.py +++ b/src/httpx/_server.py @@ -147,7 +147,7 @@ def handler(stream): ) backend = NetworkBackend() - with backend.serve("127.0.0.1", 8081, handler) as server: + with backend.serve("127.0.0.1", 8080, handler) as server: server = HTTPServer(server.host, server.port) logger.info(f"Serving on {server.url}") yield server