aiohttp: Add new aiohttp package.

Implement `aiohttp` with `ClientSession`, websockets and `SSLContext`
support.

Only client is implemented and API is mostly compatible with CPython
`aiohttp`.

Signed-off-by: Carlos Gil <carlosgilglez@gmail.com>
pull/778/head v1.22.0
Carlosgg 2023-09-05 03:56:54 +01:00 zatwierdzone przez Damien George
rodzic 57ce3ba95c
commit 7cdf708815
12 zmienionych plików z 799 dodań i 0 usunięć

Wyświetl plik

@ -0,0 +1,32 @@
aiohttp is an HTTP client module for MicroPython asyncio module,
with API mostly compatible with CPython [aiohttp](https://github.com/aio-libs/aiohttp)
module.
> [!NOTE]
> Only client is implemented.
See `examples/client.py`
```py
import aiohttp
import asyncio
async def main():
async with aiohttp.ClientSession() as session:
async with session.get('http://micropython.org') as response:
print("Status:", response.status)
print("Content-Type:", response.headers['Content-Type'])
html = await response.text()
print("Body:", html[:15], "...")
asyncio.run(main())
```
```
$ micropython examples/client.py
Status: 200
Content-Type: text/html; charset=utf-8
Body: <!DOCTYPE html> ...
```

Wyświetl plik

@ -0,0 +1,264 @@
# MicroPython aiohttp library
# MIT license; Copyright (c) 2023 Carlos Gil
import asyncio
import json as _json
from .aiohttp_ws import (
_WSRequestContextManager,
ClientWebSocketResponse,
WebSocketClient,
WSMsgType,
)
HttpVersion10 = "HTTP/1.0"
HttpVersion11 = "HTTP/1.1"
class ClientResponse:
def __init__(self, reader):
self.content = reader
def _decode(self, data):
c_encoding = self.headers.get("Content-Encoding")
if c_encoding in ("gzip", "deflate", "gzip,deflate"):
try:
import deflate, io
if c_encoding == "deflate":
with deflate.DeflateIO(io.BytesIO(data), deflate.ZLIB) as d:
return d.read()
elif c_encoding == "gzip":
with deflate.DeflateIO(io.BytesIO(data), deflate.GZIP, 15) as d:
return d.read()
except ImportError:
print("WARNING: deflate module required")
return data
async def read(self, sz=-1):
return self._decode(await self.content.read(sz))
async def text(self, encoding="utf-8"):
return (await self.read(sz=-1)).decode(encoding)
async def json(self):
return _json.loads(await self.read())
def __repr__(self):
return "<ClientResponse %d %s>" % (self.status, self.headers)
class ChunkedClientResponse(ClientResponse):
def __init__(self, reader):
self.content = reader
self.chunk_size = 0
async def read(self, sz=4 * 1024 * 1024):
if self.chunk_size == 0:
l = await self.content.readline()
l = l.split(b";", 1)[0]
self.chunk_size = int(l, 16)
if self.chunk_size == 0:
# End of message
sep = await self.content.read(2)
assert sep == b"\r\n"
return b""
data = await self.content.read(min(sz, self.chunk_size))
self.chunk_size -= len(data)
if self.chunk_size == 0:
sep = await self.content.read(2)
assert sep == b"\r\n"
return self._decode(data)
def __repr__(self):
return "<ChunkedClientResponse %d %s>" % (self.status, self.headers)
class _RequestContextManager:
def __init__(self, client, request_co):
self.reqco = request_co
self.client = client
async def __aenter__(self):
return await self.reqco
async def __aexit__(self, *args):
await self.client._reader.aclose()
return await asyncio.sleep(0)
class ClientSession:
def __init__(self, base_url="", headers={}, version=HttpVersion10):
self._reader = None
self._base_url = base_url
self._base_headers = {"Connection": "close", "User-Agent": "compat"}
self._base_headers.update(**headers)
self._http_version = version
async def __aenter__(self):
return self
async def __aexit__(self, *args):
return await asyncio.sleep(0)
# TODO: Implement timeouts
async def _request(self, method, url, data=None, json=None, ssl=None, params=None, headers={}):
redir_cnt = 0
redir_url = None
while redir_cnt < 2:
reader = await self.request_raw(method, url, data, json, ssl, params, headers)
_headers = []
sline = await reader.readline()
sline = sline.split(None, 2)
status = int(sline[1])
chunked = False
while True:
line = await reader.readline()
if not line or line == b"\r\n":
break
_headers.append(line)
if line.startswith(b"Transfer-Encoding:"):
if b"chunked" in line:
chunked = True
elif line.startswith(b"Location:"):
url = line.rstrip().split(None, 1)[1].decode("latin-1")
if 301 <= status <= 303:
redir_cnt += 1
await reader.aclose()
continue
break
if chunked:
resp = ChunkedClientResponse(reader)
else:
resp = ClientResponse(reader)
resp.status = status
resp.headers = _headers
resp.url = url
if params:
resp.url += "?" + "&".join(f"{k}={params[k]}" for k in sorted(params))
try:
resp.headers = {
val.split(":", 1)[0]: val.split(":", 1)[-1].strip()
for val in [hed.decode().strip() for hed in _headers]
}
except Exception:
pass
self._reader = reader
return resp
async def request_raw(
self,
method,
url,
data=None,
json=None,
ssl=None,
params=None,
headers={},
is_handshake=False,
version=None,
):
if json and isinstance(json, dict):
data = _json.dumps(json)
if data is not None and method == "GET":
method = "POST"
if params:
url += "?" + "&".join(f"{k}={params[k]}" for k in sorted(params))
try:
proto, dummy, host, path = url.split("/", 3)
except ValueError:
proto, dummy, host = url.split("/", 2)
path = ""
if proto == "http:":
port = 80
elif proto == "https:":
port = 443
if ssl is None:
ssl = True
else:
raise ValueError("Unsupported protocol: " + proto)
if ":" in host:
host, port = host.split(":", 1)
port = int(port)
reader, writer = await asyncio.open_connection(host, port, ssl=ssl)
# Use protocol 1.0, because 1.1 always allows to use chunked transfer-encoding
# But explicitly set Connection: close, even though this should be default for 1.0,
# because some servers misbehave w/o it.
if version is None:
version = self._http_version
if "Host" not in headers:
headers.update(Host=host)
if not data:
query = "%s /%s %s\r\n%s\r\n" % (
method,
path,
version,
"\r\n".join(f"{k}: {v}" for k, v in headers.items()) + "\r\n" if headers else "",
)
else:
headers.update(**{"Content-Length": len(str(data))})
if json:
headers.update(**{"Content-Type": "application/json"})
query = """%s /%s %s\r\n%s\r\n%s\r\n\r\n""" % (
method,
path,
version,
"\r\n".join(f"{k}: {v}" for k, v in headers.items()) + "\r\n",
data,
)
if not is_handshake:
await writer.awrite(query.encode("latin-1"))
return reader
else:
await writer.awrite(query.encode())
return reader, writer
def request(self, method, url, data=None, json=None, ssl=None, params=None, headers={}):
return _RequestContextManager(
self,
self._request(
method,
self._base_url + url,
data=data,
json=json,
ssl=ssl,
params=params,
headers=dict(**self._base_headers, **headers),
),
)
def get(self, url, **kwargs):
return self.request("GET", url, **kwargs)
def post(self, url, **kwargs):
return self.request("POST", url, **kwargs)
def put(self, url, **kwargs):
return self.request("PUT", url, **kwargs)
def patch(self, url, **kwargs):
return self.request("PATCH", url, **kwargs)
def delete(self, url, **kwargs):
return self.request("DELETE", url, **kwargs)
def head(self, url, **kwargs):
return self.request("HEAD", url, **kwargs)
def options(self, url, **kwargs):
return self.request("OPTIONS", url, **kwargs)
def ws_connect(self, url, ssl=None):
return _WSRequestContextManager(self, self._ws_connect(url, ssl=ssl))
async def _ws_connect(self, url, ssl=None):
ws_client = WebSocketClient(None)
await ws_client.connect(url, ssl=ssl, handshake_request=self.request_raw)
self._reader = ws_client.reader
return ClientWebSocketResponse(ws_client)

Wyświetl plik

@ -0,0 +1,269 @@
# MicroPython aiohttp library
# MIT license; Copyright (c) 2023 Carlos Gil
# adapted from https://github.com/danni/uwebsockets
# and https://github.com/miguelgrinberg/microdot/blob/main/src/microdot_asyncio_websocket.py
import asyncio
import random
import json as _json
import binascii
import re
import struct
from collections import namedtuple
URL_RE = re.compile(r"(wss|ws)://([A-Za-z0-9-\.]+)(?:\:([0-9]+))?(/.+)?")
URI = namedtuple("URI", ("protocol", "hostname", "port", "path")) # noqa: PYI024
def urlparse(uri):
"""Parse ws:// URLs"""
match = URL_RE.match(uri)
if match:
protocol = match.group(1)
host = match.group(2)
port = match.group(3)
path = match.group(4)
if protocol == "wss":
if port is None:
port = 443
elif protocol == "ws":
if port is None:
port = 80
else:
raise ValueError("Scheme {} is invalid".format(protocol))
return URI(protocol, host, int(port), path)
class WebSocketMessage:
def __init__(self, opcode, data):
self.type = opcode
self.data = data
class WSMsgType:
TEXT = 1
BINARY = 2
ERROR = 258
class WebSocketClient:
CONT = 0
TEXT = 1
BINARY = 2
CLOSE = 8
PING = 9
PONG = 10
def __init__(self, params):
self.params = params
self.closed = False
self.reader = None
self.writer = None
async def connect(self, uri, ssl=None, handshake_request=None):
uri = urlparse(uri)
assert uri
if uri.protocol == "wss":
if not ssl:
ssl = True
await self.handshake(uri, ssl, handshake_request)
@classmethod
def _parse_frame_header(cls, header):
byte1, byte2 = struct.unpack("!BB", header)
# Byte 1: FIN(1) _(1) _(1) _(1) OPCODE(4)
fin = bool(byte1 & 0x80)
opcode = byte1 & 0x0F
# Byte 2: MASK(1) LENGTH(7)
mask = bool(byte2 & (1 << 7))
length = byte2 & 0x7F
return fin, opcode, mask, length
def _process_websocket_frame(self, opcode, payload):
if opcode == self.TEXT:
payload = payload.decode()
elif opcode == self.BINARY:
pass
elif opcode == self.CLOSE:
# raise OSError(32, "Websocket connection closed")
return opcode, payload
elif opcode == self.PING:
return self.PONG, payload
elif opcode == self.PONG: # pragma: no branch
return None, None
return None, payload
@classmethod
def _encode_websocket_frame(cls, opcode, payload):
if opcode == cls.TEXT:
payload = payload.encode()
length = len(payload)
fin = mask = True
# Frame header
# Byte 1: FIN(1) _(1) _(1) _(1) OPCODE(4)
byte1 = 0x80 if fin else 0
byte1 |= opcode
# Byte 2: MASK(1) LENGTH(7)
byte2 = 0x80 if mask else 0
if length < 126: # 126 is magic value to use 2-byte length header
byte2 |= length
frame = struct.pack("!BB", byte1, byte2)
elif length < (1 << 16): # Length fits in 2-bytes
byte2 |= 126 # Magic code
frame = struct.pack("!BBH", byte1, byte2, length)
elif length < (1 << 64):
byte2 |= 127 # Magic code
frame = struct.pack("!BBQ", byte1, byte2, length)
else:
raise ValueError
# Mask is 4 bytes
mask_bits = struct.pack("!I", random.getrandbits(32))
frame += mask_bits
payload = bytes(b ^ mask_bits[i % 4] for i, b in enumerate(payload))
return frame + payload
async def handshake(self, uri, ssl, req):
headers = {}
_http_proto = "http" if uri.protocol != "wss" else "https"
url = f"{_http_proto}://{uri.hostname}:{uri.port}{uri.path or '/'}"
key = binascii.b2a_base64(bytes(random.getrandbits(8) for _ in range(16)))[:-1]
headers["Host"] = f"{uri.hostname}:{uri.port}"
headers["Connection"] = "Upgrade"
headers["Upgrade"] = "websocket"
headers["Sec-WebSocket-Key"] = key
headers["Sec-WebSocket-Version"] = "13"
headers["Origin"] = f"{_http_proto}://{uri.hostname}:{uri.port}"
self.reader, self.writer = await req(
"GET",
url,
ssl=ssl,
headers=headers,
is_handshake=True,
version="HTTP/1.1",
)
header = await self.reader.readline()
header = header[:-2]
assert header.startswith(b"HTTP/1.1 101 "), header
while header:
header = await self.reader.readline()
header = header[:-2]
async def receive(self):
while True:
opcode, payload = await self._read_frame()
send_opcode, data = self._process_websocket_frame(opcode, payload)
if send_opcode: # pragma: no cover
await self.send(data, send_opcode)
if opcode == self.CLOSE:
self.closed = True
return opcode, data
elif data: # pragma: no branch
return opcode, data
async def send(self, data, opcode=None):
frame = self._encode_websocket_frame(
opcode or (self.TEXT if isinstance(data, str) else self.BINARY), data
)
self.writer.write(frame)
await self.writer.drain()
async def close(self):
if not self.closed: # pragma: no cover
self.closed = True
await self.send(b"", self.CLOSE)
async def _read_frame(self):
header = await self.reader.read(2)
if len(header) != 2: # pragma: no cover
# raise OSError(32, "Websocket connection closed")
opcode = self.CLOSE
payload = b""
return opcode, payload
fin, opcode, has_mask, length = self._parse_frame_header(header)
if length == 126: # Magic number, length header is 2 bytes
(length,) = struct.unpack("!H", await self.reader.read(2))
elif length == 127: # Magic number, length header is 8 bytes
(length,) = struct.unpack("!Q", await self.reader.read(8))
if has_mask: # pragma: no cover
mask = await self.reader.read(4)
payload = await self.reader.read(length)
if has_mask: # pragma: no cover
payload = bytes(x ^ mask[i % 4] for i, x in enumerate(payload))
return opcode, payload
class ClientWebSocketResponse:
def __init__(self, wsclient):
self.ws = wsclient
def __aiter__(self):
return self
async def __anext__(self):
msg = WebSocketMessage(*await self.ws.receive())
# print(msg.data, msg.type) # DEBUG
if (not msg.data and msg.type == self.ws.CLOSE) or self.ws.closed:
raise StopAsyncIteration
return msg
async def close(self):
await self.ws.close()
async def send_str(self, data):
if not isinstance(data, str):
raise TypeError("data argument must be str (%r)" % type(data))
await self.ws.send(data)
async def send_bytes(self, data):
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError("data argument must be byte-ish (%r)" % type(data))
await self.ws.send(data)
async def send_json(self, data):
await self.send_str(_json.dumps(data))
async def receive_str(self):
msg = WebSocketMessage(*await self.ws.receive())
if msg.type != self.ws.TEXT:
raise TypeError(f"Received message {msg.type}:{msg.data!r} is not str")
return msg.data
async def receive_bytes(self):
msg = WebSocketMessage(*await self.ws.receive())
if msg.type != self.ws.BINARY:
raise TypeError(f"Received message {msg.type}:{msg.data!r} is not bytes")
return msg.data
async def receive_json(self):
data = await self.receive_str()
return _json.loads(data)
class _WSRequestContextManager:
def __init__(self, client, request_co):
self.reqco = request_co
self.client = client
async def __aenter__(self):
return await self.reqco
async def __aexit__(self, *args):
await self.client._reader.aclose()
return await asyncio.sleep(0)

Wyświetl plik

@ -0,0 +1,18 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
async def main():
async with aiohttp.ClientSession() as session:
async with session.get("http://micropython.org") as response:
print("Status:", response.status)
print("Content-Type:", response.headers["Content-Type"])
html = await response.text()
print("Body:", html[:15], "...")
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,20 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
headers = {"Accept-Encoding": "gzip,deflate"}
async def main():
async with aiohttp.ClientSession(headers=headers, version=aiohttp.HttpVersion11) as session:
async with session.get("http://micropython.org") as response:
print("Status:", response.status)
print("Content-Type:", response.headers["Content-Type"])
print(response.headers)
html = await response.text()
print(html)
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,29 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
URL = sys.argv.pop()
if not URL.startswith("http"):
URL = "http://micropython.org"
print(URL)
async def fetch(client):
async with client.get(URL) as resp:
assert resp.status == 200
return await resp.text()
async def main():
async with aiohttp.ClientSession() as client:
html = await fetch(client)
print(html)
if __name__ == "__main__":
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,18 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
headers = {"Authorization": "Basic bG9naW46cGFzcw=="}
async def main():
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get("http://httpbin.org/headers") as r:
json_body = await r.json()
print(json_body)
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,25 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
async def main():
async with aiohttp.ClientSession("http://httpbin.org") as session:
async with session.get("/get") as resp:
assert resp.status == 200
rget = await resp.text()
print(f"GET: {rget}")
async with session.post("/post", json={"foo": "bar"}) as resp:
assert resp.status == 200
rpost = await resp.text()
print(f"POST: {rpost}")
async with session.put("/put", data=b"data") as resp:
assert resp.status == 200
rput = await resp.json()
print("PUT: ", rput)
if __name__ == "__main__":
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,20 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
params = {"key1": "value1", "key2": "value2"}
async def main():
async with aiohttp.ClientSession() as session:
async with session.get("http://httpbin.org/get", params=params) as response:
expect = "http://httpbin.org/get?key1=value1&key2=value2"
assert str(response.url) == expect, f"{response.url} != {expect}"
html = await response.text()
print(html)
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,44 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
try:
URL = sys.argv[1] # expects a websocket echo server
except Exception:
URL = "ws://echo.websocket.events"
sslctx = False
if URL.startswith("wss:"):
try:
import ssl
sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
sslctx.verify_mode = ssl.CERT_NONE
except Exception:
pass
async def ws_test_echo(session):
async with session.ws_connect(URL, ssl=sslctx) as ws:
await ws.send_str("hello world!\r\n")
async for msg in ws:
if msg.type == aiohttp.WSMsgType.TEXT:
print(msg.data)
if "close" in msg.data:
break
await ws.send_str("close\r\n")
await ws.close()
async def main():
async with aiohttp.ClientSession() as session:
await ws_test_echo(session)
if __name__ == "__main__":
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,53 @@
import sys
sys.path.insert(0, ".")
import aiohttp
import asyncio
try:
URL = sys.argv[1] # expects a websocket echo server
READ_BANNER = False
except Exception:
URL = "ws://echo.websocket.events"
READ_BANNER = True
sslctx = False
if URL.startswith("wss:"):
try:
import ssl
sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
sslctx.verify_mode = ssl.CERT_NONE
except Exception:
pass
async def ws_test_echo(session):
async with session.ws_connect(URL, ssl=sslctx) as ws:
if READ_BANNER:
print(await ws.receive_str())
try:
while True:
await ws.send_str(f"{input('>>> ')}\r\n")
async for msg in ws:
if msg.type == aiohttp.WSMsgType.TEXT:
print(msg.data, end="")
break
except KeyboardInterrupt:
pass
finally:
await ws.close()
async def main():
async with aiohttp.ClientSession() as session:
await ws_test_echo(session)
if __name__ == "__main__":
asyncio.run(main())

Wyświetl plik

@ -0,0 +1,7 @@
metadata(
description="HTTP client module for MicroPython asyncio module",
version="0.0.1",
pypi="aiohttp",
)
package("aiohttp")