diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ca355b..7617e1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,21 +10,24 @@ jobs: tox: runs-on: ubuntu-latest strategy: - max-parallel: 5 + max-parallel: 7 matrix: python-version: - - 3.7 - 3.8 - 3.9 - "3.10" - - pypy-3.7 - - pypy-3.8 + - "3.11" + - "3.12" + - "3.13" + - pypy-3.9 + - pypy-3.10 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install tox run: | python -m pip install --upgrade pip setuptools @@ -35,6 +38,6 @@ jobs: - name: Test with tox run: | tox --parallel 0 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v4 with: file: ./coverage.xml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..38d4fcc --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,17 @@ +version: 2 + +build: + os: ubuntu-22.04 + apt_packages: + - graphviz + tools: + python: "3.8" + +sphinx: + configuration: docs/source/conf.py + +python: + install: + - method: pip + path: . + - requirements: docs/requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index d2baf3f..f2f65de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include LICENSE.txt README.rst notes.org tiny-client-demo.py h11/py.typed recursive-include docs * -recursive-include h11/tests/data * +recursive-include h11/tests * recursive-include fuzz * prune docs/build diff --git a/README.rst b/README.rst index 56e277e..5f28616 100644 --- a/README.rst +++ b/README.rst @@ -112,7 +112,7 @@ library. It has a test suite with 100.0% coverage for both statements and branches. -Currently it supports Python 3 (testing on 3.7-3.10) and PyPy 3. +Currently it supports Python 3 (testing on 3.8-3.12) and PyPy 3. The last Python 2-compatible version was h11 0.11.x. (Originally it had a Cython wrapper for `http-parser `_ and a beautiful nested state diff --git a/bench/benchmarks/benchmarks.py b/bench/benchmarks/benchmarks.py index abc0079..73d078e 100644 --- a/bench/benchmarks/benchmarks.py +++ b/bench/benchmarks/benchmarks.py @@ -60,7 +60,7 @@ def _run_basic_get_repeatedly(): for _ in range(REPEAT): time_server_basic_get_with_realistic_headers() finish = default_timer() - print("{:.1f} requests/sec".format(REPEAT / (finish - start))) + print(f"{REPEAT / (finish - start):.1f} requests/sec") if __name__ == "__main__": diff --git a/docs/requirements.txt b/docs/requirements.txt index b33e9c4..1c6aca5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ mistune jsonschema ipython +sphinx<4 +jinja2<3 +markupsafe<2 diff --git a/docs/source/changes.rst b/docs/source/changes.rst index 98540b3..db234fd 100644 --- a/docs/source/changes.rst +++ b/docs/source/changes.rst @@ -5,6 +5,33 @@ History of changes .. towncrier release notes start +H11 0.16.0 (2025-04-23) +----------------------- + +Security fix +~~~~~~~~~~~~ + +Reject certain malformed `Transfer-Encoding: chunked` bodies that were previously accepted. These could have enabled request-smuggling attacks when an h11-based HTTP server was placed behind a load balancer with a matching bug in its `chunked` handling. + +Advisory with more details: https://github.com/python-hyper/h11/security/advisories/GHSA-vqfr-h8mv-ghfj + +Reported by: Jeppe Bonde Weikop + +H11 0.15.0 (2025-04-23) +----------------------- + +Bugfixes +~~~~~~~~ + +- Reject Content-Lengths >= 1 zettabyte (1 billion terabytes) early, `without attempting to parse the integer `__ (`#178 `__) + + +Miscellaneous internal changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Remove the `tests` folder from wheel files. This reduces the zipped file size by 20KB (about 30%). (`#158 `__) + + H11 0.14.0 (2022-09-25) ----------------------- diff --git a/docs/source/conf.py b/docs/source/conf.py index 0d8b494..b3627f5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # h11 documentation build configuration file, created by # sphinx-quickstart on Tue May 3 00:20:14 2016. diff --git a/docs/source/index.rst b/docs/source/index.rst index dd4d733..ee02847 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -44,7 +44,7 @@ whatever. But h11 makes it much easier to implement something like Vital statistics ---------------- -* Requirements: Python 3.7+ (PyPy works great) +* Requirements: Python 3.8+ (PyPy works great) The last Python 2-compatible version was h11 0.11.x. diff --git a/docs/source/make-state-diagrams.py b/docs/source/make-state-diagrams.py index 16f033e..617efa5 100644 --- a/docs/source/make-state-diagrams.py +++ b/docs/source/make-state-diagrams.py @@ -38,16 +38,16 @@ def __init__(self): def e(self, source, target, label, color, italicize=False, weight=1): if italicize: - quoted_label = "<{}>".format(label) + quoted_label = f"<{label}>" else: - quoted_label = '<{}>'.format(label) + quoted_label = f'<{label}>' self.edges.append( - '{source} -> {target} [\n' - ' label={quoted_label},\n' - ' color="{color}", fontcolor="{color}",\n' - ' weight={weight},\n' - ']\n' - .format(**locals())) + f'{source} -> {target} [\n' + f' label={quoted_label},\n' + f' color="{color}", fontcolor="{color}",\n' + f' weight={weight},\n' + f']\n' + ) def write(self, f): self.edges.sort() @@ -150,7 +150,7 @@ def make_dot(role, out_path): else: (their_state, our_state) = state_pair edges.e(our_state, updates[role], - "peer in
{}".format(their_state), + f"peer in
{their_state}", color=_STATE_COLOR) if role is CLIENT: diff --git a/examples/trio-server.py b/examples/trio-server.py index 361a288..996afb6 100644 --- a/examples/trio-server.py +++ b/examples/trio-server.py @@ -106,6 +106,7 @@ def format_date_time(dt=None): # I/O adapter: h11 <-> trio ################################################################ + # The core of this could be factored out to be usable for trio-based clients # too, as well as servers. But as a simplified pedagogical example we don't # attempt this here. @@ -117,7 +118,7 @@ def __init__(self, stream): self.conn = h11.Connection(h11.SERVER) # Our Server: header self.ident = " ".join( - ["h11-example-trio-server/{}".format(h11.__version__), h11.PRODUCT_ID] + [f"h11-example-trio-server/{h11.__version__}", h11.PRODUCT_ID] ).encode("ascii") # A unique id for this connection, to include in debugging output # (useful for understanding what's going on if there are multiple @@ -205,13 +206,14 @@ def basic_headers(self): def info(self, *args): # Little debugging method - print("{}:".format(self._obj_id), *args) + print(f"{self._obj_id}:", *args) ################################################################ # Server main loop ################################################################ + # General theory: # # If everything goes well: @@ -251,7 +253,7 @@ async def http_serve(stream): if type(event) is h11.Request: await send_echo_response(wrapper, event) except Exception as exc: - wrapper.info("Error during response handler: {!r}".format(exc)) + wrapper.info(f"Error during response handler: {exc!r}") await maybe_send_error_response(wrapper, exc) if wrapper.conn.our_state is h11.MUST_CLOSE: @@ -266,7 +268,7 @@ async def http_serve(stream): states = wrapper.conn.states wrapper.info("unexpected state", states, "-- bailing out") await maybe_send_error_response( - wrapper, RuntimeError("unexpected state {}".format(states)) + wrapper, RuntimeError(f"unexpected state {states}") ) await wrapper.shutdown_and_clean_up() return @@ -276,6 +278,7 @@ async def http_serve(stream): # Actual response handlers ################################################################ + # Helper function async def send_simple_response(wrapper, status_code, content_type, body): wrapper.info("Sending", status_code, "response with", len(body), "bytes") @@ -340,7 +343,7 @@ async def send_echo_response(wrapper, request): async def serve(port): - print("listening on http://localhost:{}".format(port)) + print(f"listening on http://localhost:{port}") try: await trio.serve_tcp(http_serve, port) except KeyboardInterrupt: diff --git a/format-requirements.txt b/format-requirements.txt new file mode 100644 index 0000000..a45e8c9 --- /dev/null +++ b/format-requirements.txt @@ -0,0 +1,2 @@ +black==23.3.0 +isort==5.12.0 \ No newline at end of file diff --git a/h11/_connection.py b/h11/_connection.py index d175270..e37d82a 100644 --- a/h11/_connection.py +++ b/h11/_connection.py @@ -1,6 +1,17 @@ # This contains the main Connection class. Everything in h11 revolves around # this. -from typing import Any, Callable, cast, Dict, List, Optional, Tuple, Type, Union +from typing import ( + Any, + Callable, + cast, + Dict, + List, + Optional, + overload, + Tuple, + Type, + Union, +) from ._events import ( ConnectionClosed, @@ -57,6 +68,7 @@ class PAUSED(Sentinel, metaclass=Sentinel): # - Apache: <8 KiB per line> DEFAULT_MAX_INCOMPLETE_EVENT_SIZE = 16 * 1024 + # RFC 7230's rules for connection lifecycles: # - If either side says they want to close the connection, then the connection # must close. @@ -160,7 +172,7 @@ def __init__( self._max_incomplete_event_size = max_incomplete_event_size # State and role tracking if our_role not in (CLIENT, SERVER): - raise ValueError("expected CLIENT or SERVER, not {!r}".format(our_role)) + raise ValueError(f"expected CLIENT or SERVER, not {our_role!r}") self.our_role = our_role self.their_role: Type[Sentinel] if our_role is CLIENT: @@ -416,7 +428,7 @@ def _extract_next_receive_event( # return that event, and then the state will change and we'll # get called again to generate the actual ConnectionClosed(). if hasattr(self._reader, "read_eof"): - event = self._reader.read_eof() # type: ignore[attr-defined] + event = self._reader.read_eof() else: event = ConnectionClosed() if event is None: @@ -488,6 +500,20 @@ def next_event(self) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: else: raise + @overload + def send(self, event: ConnectionClosed) -> None: + ... + + @overload + def send( + self, event: Union[Request, InformationalResponse, Response, Data, EndOfMessage] + ) -> bytes: + ... + + @overload + def send(self, event: Event) -> Optional[bytes]: + ... + def send(self, event: Event) -> Optional[bytes]: """Convert a high-level event into bytes that can be sent to the peer, while updating our internal state machine. diff --git a/h11/_events.py b/h11/_events.py index 075bf8a..ca1c3ad 100644 --- a/h11/_events.py +++ b/h11/_events.py @@ -7,8 +7,8 @@ import re from abc import ABC -from dataclasses import dataclass, field -from typing import Any, cast, Dict, List, Tuple, Union +from dataclasses import dataclass +from typing import List, Tuple, Union from ._abnf import method, request_target from ._headers import Headers, normalize_and_validate diff --git a/h11/_headers.py b/h11/_headers.py index b97d020..31da3e2 100644 --- a/h11/_headers.py +++ b/h11/_headers.py @@ -12,6 +12,8 @@ except ImportError: from typing_extensions import Literal # type: ignore +CONTENT_LENGTH_MAX_DIGITS = 20 # allow up to 1 billion TB - 1 + # Facts # ----- @@ -173,6 +175,8 @@ def normalize_and_validate( raise LocalProtocolError("conflicting Content-Length headers") value = lengths.pop() validate(_content_length_re, value, "bad Content-Length") + if len(value) > CONTENT_LENGTH_MAX_DIGITS: + raise LocalProtocolError("bad Content-Length") if seen_content_length is None: seen_content_length = value new_headers.append((raw_name, name, value)) diff --git a/h11/_readers.py b/h11/_readers.py index 08a9574..576804c 100644 --- a/h11/_readers.py +++ b/h11/_readers.py @@ -148,10 +148,9 @@ def read_eof(self) -> NoReturn: class ChunkedReader: def __init__(self) -> None: self._bytes_in_chunk = 0 - # After reading a chunk, we have to throw away the trailing \r\n; if - # this is >0 then we discard that many bytes before resuming regular - # de-chunkification. - self._bytes_to_discard = 0 + # After reading a chunk, we have to throw away the trailing \r\n. + # This tracks the bytes that we need to match and throw away. + self._bytes_to_discard = b"" self._reading_trailer = False def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: @@ -160,15 +159,19 @@ def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: if lines is None: return None return EndOfMessage(headers=list(_decode_header_lines(lines))) - if self._bytes_to_discard > 0: - data = buf.maybe_extract_at_most(self._bytes_to_discard) + if self._bytes_to_discard: + data = buf.maybe_extract_at_most(len(self._bytes_to_discard)) if data is None: return None - self._bytes_to_discard -= len(data) - if self._bytes_to_discard > 0: + if data != self._bytes_to_discard[: len(data)]: + raise LocalProtocolError( + f"malformed chunk footer: {data!r} (expected {self._bytes_to_discard!r})" + ) + self._bytes_to_discard = self._bytes_to_discard[len(data) :] + if self._bytes_to_discard: return None # else, fall through and read some more - assert self._bytes_to_discard == 0 + assert self._bytes_to_discard == b"" if self._bytes_in_chunk == 0: # We need to refill our chunk count chunk_header = buf.maybe_extract_next_line() @@ -194,7 +197,7 @@ def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: return None self._bytes_in_chunk -= len(data) if self._bytes_in_chunk == 0: - self._bytes_to_discard = 2 + self._bytes_to_discard = b"\r\n" chunk_end = True else: chunk_end = False diff --git a/h11/_state.py b/h11/_state.py index 3593430..3ad444b 100644 --- a/h11/_state.py +++ b/h11/_state.py @@ -283,9 +283,7 @@ def process_event( assert role is SERVER if server_switch_event not in self.pending_switch_proposals: raise LocalProtocolError( - "Received server {} event without a pending proposal".format( - server_switch_event - ) + "Received server _SWITCH_UPGRADE event without a pending proposal" ) _event_type = (event_type, server_switch_event) if server_switch_event is None and _event_type is Response: @@ -358,7 +356,7 @@ def _fire_state_triggered_transitions(self) -> None: def start_next_cycle(self) -> None: if self.states != {CLIENT: DONE, SERVER: DONE}: raise LocalProtocolError( - "not in a reusable state. self.states={}".format(self.states) + f"not in a reusable state. self.states={self.states}" ) # Can't reach DONE/DONE with any of these active, but still, let's be # sure. diff --git a/h11/_version.py b/h11/_version.py index 4c89113..76e7327 100644 --- a/h11/_version.py +++ b/h11/_version.py @@ -13,4 +13,4 @@ # want. (Contrast with the special suffix 1.0.0.dev, which sorts *before* # 1.0.0.) -__version__ = "0.14.0" +__version__ = "0.16.0" diff --git a/h11/tests/test_against_stdlib_http.py b/h11/tests/test_against_stdlib_http.py index d2ee131..3f66a10 100644 --- a/h11/tests/test_against_stdlib_http.py +++ b/h11/tests/test_against_stdlib_http.py @@ -13,7 +13,7 @@ @contextmanager def socket_server( - handler: Callable[..., socketserver.BaseRequestHandler] + handler: Callable[..., socketserver.BaseRequestHandler], ) -> Generator[socketserver.TCPServer, None, None]: httpd = socketserver.TCPServer(("127.0.0.1", 0), handler) thread = threading.Thread( @@ -39,17 +39,17 @@ def translate_path(self, path: str) -> str: def test_h11_as_client() -> None: with socket_server(SingleMindedRequestHandler) as httpd: - with closing(socket.create_connection(httpd.server_address)) as s: + with closing(socket.create_connection(httpd.server_address)) as s: # type: ignore[arg-type] c = h11.Connection(h11.CLIENT) s.sendall( - c.send( # type: ignore[arg-type] + c.send( h11.Request( method="GET", target="/foo", headers=[("Host", "localhost")] ) ) ) - s.sendall(c.send(h11.EndOfMessage())) # type: ignore[arg-type] + s.sendall(c.send(h11.EndOfMessage())) data = bytearray() while True: @@ -96,7 +96,7 @@ def handle(self) -> None: }, } ) - s.sendall(c.send(h11.Response(status_code=200, headers=[]))) # type: ignore[arg-type] + s.sendall(c.send(h11.Response(status_code=200, headers=[]))) s.sendall(c.send(h11.Data(data=info.encode("ascii")))) s.sendall(c.send(h11.EndOfMessage())) @@ -104,7 +104,7 @@ def handle(self) -> None: def test_h11_as_server() -> None: with socket_server(H11RequestHandler) as httpd: host, port = httpd.server_address - url = "http://{}:{}/some-path".format(host, port) + url = f"http://{host}:{port}/some-path" # type: ignore[str-bytes-safe] with closing(urlopen(url)) as f: assert f.getcode() == 200 data = f.read() diff --git a/h11/tests/test_connection.py b/h11/tests/test_connection.py index 73a27b9..01260dc 100644 --- a/h11/tests/test_connection.py +++ b/h11/tests/test_connection.py @@ -7,7 +7,6 @@ ConnectionClosed, Data, EndOfMessage, - Event, InformationalResponse, Request, Response, @@ -17,7 +16,6 @@ CLOSED, DONE, ERROR, - IDLE, MIGHT_SWITCH_PROTOCOL, MUST_CLOSE, SEND_BODY, @@ -48,15 +46,15 @@ def test__keep_alive() -> None: ) ) assert not _keep_alive( - Request(method="GET", target="/", headers=[], http_version="1.0") # type: ignore[arg-type] + Request(method="GET", target="/", headers=[], http_version="1.0") ) - assert _keep_alive(Response(status_code=200, headers=[])) # type: ignore[arg-type] + assert _keep_alive(Response(status_code=200, headers=[])) assert not _keep_alive(Response(status_code=200, headers=[("Connection", "close")])) assert not _keep_alive( Response(status_code=200, headers=[("Connection", "a, b, cLOse, foo")]) ) - assert not _keep_alive(Response(status_code=200, headers=[], http_version="1.0")) # type: ignore[arg-type] + assert not _keep_alive(Response(status_code=200, headers=[], http_version="1.0")) def test__body_framing() -> None: @@ -135,7 +133,7 @@ def test_Connection_basics_and_content_length() -> None: assert p.conn[CLIENT].their_http_version is None assert p.conn[SERVER].their_http_version == b"1.1" - data = p.send(SERVER, InformationalResponse(status_code=100, headers=[])) # type: ignore[arg-type] + data = p.send(SERVER, InformationalResponse(status_code=100, headers=[])) assert data == b"HTTP/1.1 100 \r\n\r\n" data = p.send(SERVER, Response(status_code=200, headers=[("Content-Length", "11")])) @@ -247,7 +245,7 @@ def test_client_talking_to_http10_server() -> None: assert c.our_state is DONE # No content-length, so Http10 framing for body assert receive_and_get(c, b"HTTP/1.0 200 OK\r\n\r\n") == [ - Response(status_code=200, headers=[], http_version="1.0", reason=b"OK") # type: ignore[arg-type] + Response(status_code=200, headers=[], http_version="1.0", reason=b"OK") ] assert c.our_state is MUST_CLOSE assert receive_and_get(c, b"12345") == [Data(data=b"12345")] @@ -261,14 +259,14 @@ def test_server_talking_to_http10_client() -> None: # No content-length, so no body # NB: no host header assert receive_and_get(c, b"GET / HTTP/1.0\r\n\r\n") == [ - Request(method="GET", target="/", headers=[], http_version="1.0"), # type: ignore[arg-type] + Request(method="GET", target="/", headers=[], http_version="1.0"), EndOfMessage(), ] assert c.their_state is MUST_CLOSE # We automatically Connection: close back at them assert ( - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) == b"HTTP/1.1 200 \r\nConnection: close\r\n\r\n" ) @@ -356,7 +354,7 @@ def test_automagic_connection_close_handling() -> None: p.send( SERVER, # no header here... - [Response(status_code=204, headers=[]), EndOfMessage()], # type: ignore[arg-type] + [Response(status_code=204, headers=[]), EndOfMessage()], # ...but oh look, it arrived anyway expect=[ Response(status_code=204, headers=[("connection", "close")]), @@ -390,7 +388,7 @@ def setup() -> ConnectionPair: # Disabled by 100 Continue p = setup() - p.send(SERVER, InformationalResponse(status_code=100, headers=[])) # type: ignore[arg-type] + p.send(SERVER, InformationalResponse(status_code=100, headers=[])) for conn in p.conns: assert not conn.client_is_waiting_for_100_continue assert not conn.they_are_waiting_for_100_continue @@ -471,7 +469,7 @@ def test_max_incomplete_event_size_countermeasure() -> None: # Even more data comes in, still no problem c.receive_data(b"X" * 1000) # We can respond and reuse to get the second pipelined request - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) c.send(EndOfMessage()) c.start_next_cycle() assert get_all_events(c) == [ @@ -481,7 +479,7 @@ def test_max_incomplete_event_size_countermeasure() -> None: # But once we unpause and try to read the next message, and find that it's # incomplete and the buffer is *still* way too large, then *that's* a # problem: - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) c.send(EndOfMessage()) c.start_next_cycle() with pytest.raises(RemoteProtocolError): @@ -547,7 +545,7 @@ def test_pipelining() -> None: assert c.next_event() is PAUSED - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) c.send(EndOfMessage()) assert c.their_state is DONE assert c.our_state is DONE @@ -564,7 +562,7 @@ def test_pipelining() -> None: EndOfMessage(), ] assert c.next_event() is PAUSED - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) c.send(EndOfMessage()) c.start_next_cycle() @@ -574,7 +572,7 @@ def test_pipelining() -> None: ] # Doesn't pause this time, no trailing data assert c.next_event() is NEED_DATA - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) c.send(EndOfMessage()) # Arrival of more data triggers pause @@ -594,7 +592,7 @@ def test_pipelining() -> None: def test_protocol_switch() -> None: - for (req, deny, accept) in [ + for req, deny, accept in [ ( Request( method="CONNECT", @@ -683,7 +681,7 @@ def setup() -> ConnectionPair: sc.send(EndOfMessage()) sc.start_next_cycle() assert get_all_events(sc) == [ - Request(method="GET", target="/", headers=[], http_version="1.0"), # type: ignore[arg-type] + Request(method="GET", target="/", headers=[], http_version="1.0"), EndOfMessage(), ] @@ -721,7 +719,7 @@ def setup() -> ConnectionPair: def test_close_simple() -> None: # Just immediately closing a new connection without anything having # happened yet. - for (who_shot_first, who_shot_second) in [(CLIENT, SERVER), (SERVER, CLIENT)]: + for who_shot_first, who_shot_second in [(CLIENT, SERVER), (SERVER, CLIENT)]: def setup() -> ConnectionPair: p = ConnectionPair() @@ -845,7 +843,7 @@ def test_pipelined_close() -> None: EndOfMessage(), ] assert c.states[CLIENT] is DONE - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) c.send(EndOfMessage()) assert c.states[SERVER] is DONE c.start_next_cycle() @@ -860,7 +858,7 @@ def test_pipelined_close() -> None: ConnectionClosed(), ] assert c.states == {CLIENT: CLOSED, SERVER: SEND_RESPONSE} - c.send(Response(status_code=200, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) c.send(EndOfMessage()) assert c.states == {CLIENT: CLOSED, SERVER: MUST_CLOSE} c.send(ConnectionClosed()) @@ -879,7 +877,7 @@ def setup( ) -> Tuple[Connection, Optional[List[bytes]]]: c = Connection(SERVER) receive_and_get( - c, "GET / HTTP/{}\r\nHost: a\r\n\r\n".format(http_version).encode("ascii") + c, f"GET / HTTP/{http_version}\r\nHost: a\r\n\r\n".encode("ascii") ) headers = [] if header: @@ -919,7 +917,7 @@ def test_errors() -> None: # But we can still yell at the client for sending us gibberish if role is SERVER: assert ( - c.send(Response(status_code=400, headers=[])) # type: ignore[arg-type] + c.send(Response(status_code=400, headers=[])) == b"HTTP/1.1 400 \r\nConnection: close\r\n\r\n" ) @@ -946,8 +944,8 @@ def conn(role: Type[Sentinel]) -> Connection: http_version="1.0", ) elif role is SERVER: - good = Response(status_code=200, headers=[]) # type: ignore[arg-type,assignment] - bad = Response(status_code=200, headers=[], http_version="1.0") # type: ignore[arg-type,assignment] + good = Response(status_code=200, headers=[]) # type: ignore[assignment] + bad = Response(status_code=200, headers=[], http_version="1.0") # type: ignore[assignment] # Make sure 'good' actually is good c = conn(role) c.send(good) @@ -1063,14 +1061,14 @@ def setup(method: bytes, http_version: bytes) -> Connection: # No Content-Length, HTTP/1.1 peer, should use chunked c = setup(method, b"1.1") assert ( - c.send(Response(status_code=200, headers=[])) == b"HTTP/1.1 200 \r\n" # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) == b"HTTP/1.1 200 \r\n" b"Transfer-Encoding: chunked\r\n\r\n" ) # No Content-Length, HTTP/1.0 peer, frame with connection: close c = setup(method, b"1.0") assert ( - c.send(Response(status_code=200, headers=[])) == b"HTTP/1.1 200 \r\n" # type: ignore[arg-type] + c.send(Response(status_code=200, headers=[])) == b"HTTP/1.1 200 \r\n" b"Connection: close\r\n\r\n" ) diff --git a/h11/tests/test_events.py b/h11/tests/test_events.py index bc6c313..d691545 100644 --- a/h11/tests/test_events.py +++ b/h11/tests/test_events.py @@ -2,12 +2,10 @@ import pytest -from .. import _events from .._events import ( ConnectionClosed, Data, EndOfMessage, - Event, InformationalResponse, Request, Response, @@ -101,13 +99,13 @@ def test_events() -> None: with pytest.raises(LocalProtocolError): InformationalResponse(status_code=200, headers=[("Host", "a")]) - resp = Response(status_code=204, headers=[], http_version="1.0") # type: ignore[arg-type] + resp = Response(status_code=204, headers=[], http_version="1.0") assert resp.status_code == 204 assert resp.headers == [] assert resp.http_version == b"1.0" with pytest.raises(LocalProtocolError): - resp = Response(status_code=100, headers=[], http_version="1.0") # type: ignore[arg-type] + resp = Response(status_code=100, headers=[], http_version="1.0") with pytest.raises(LocalProtocolError): Response(status_code="100", headers=[], http_version="1.0") # type: ignore[arg-type] @@ -128,7 +126,7 @@ def test_events() -> None: def test_intenum_status_code() -> None: # https://github.com/python-hyper/h11/issues/72 - r = Response(status_code=HTTPStatus.OK, headers=[], http_version="1.0") # type: ignore[arg-type] + r = Response(status_code=HTTPStatus.OK, headers=[], http_version="1.0") assert r.status_code == HTTPStatus.OK assert type(r.status_code) is not type(HTTPStatus.OK) assert type(r.status_code) is int diff --git a/h11/tests/test_headers.py b/h11/tests/test_headers.py index ba53d08..b57274c 100644 --- a/h11/tests/test_headers.py +++ b/h11/tests/test_headers.py @@ -74,6 +74,8 @@ def test_normalize_and_validate() -> None: ) with pytest.raises(LocalProtocolError): normalize_and_validate([("Content-Length", "1 , 1,2")]) + with pytest.raises(LocalProtocolError): + normalize_and_validate([("Content-Length", "1" * 21)]) # 1 billion TB # transfer-encoding assert normalize_and_validate([("Transfer-Encoding", "chunked")]) == [ diff --git a/h11/tests/test_helpers.py b/h11/tests/test_helpers.py index c329c76..9a30dc6 100644 --- a/h11/tests/test_helpers.py +++ b/h11/tests/test_helpers.py @@ -1,12 +1,4 @@ -from .._events import ( - ConnectionClosed, - Data, - EndOfMessage, - Event, - InformationalResponse, - Request, - Response, -) +from .._events import Data, EndOfMessage, Response from .helpers import normalize_data_events @@ -15,7 +7,7 @@ def test_normalize_data_events() -> None: [ Data(data=bytearray(b"1")), Data(data=b"2"), - Response(status_code=200, headers=[]), # type: ignore[arg-type] + Response(status_code=200, headers=[]), Data(data=b"3"), Data(data=b"4"), EndOfMessage(), @@ -25,7 +17,7 @@ def test_normalize_data_events() -> None: ] ) == [ Data(data=b"12"), - Response(status_code=200, headers=[]), # type: ignore[arg-type] + Response(status_code=200, headers=[]), Data(data=b"34"), EndOfMessage(), Data(data=b"567"), diff --git a/h11/tests/test_io.py b/h11/tests/test_io.py index 2b47c0e..407e044 100644 --- a/h11/tests/test_io.py +++ b/h11/tests/test_io.py @@ -3,7 +3,6 @@ import pytest from .._events import ( - ConnectionClosed, Data, EndOfMessage, Event, @@ -20,18 +19,7 @@ READERS, ) from .._receivebuffer import ReceiveBuffer -from .._state import ( - CLIENT, - CLOSED, - DONE, - IDLE, - MIGHT_SWITCH_PROTOCOL, - MUST_CLOSE, - SEND_BODY, - SEND_RESPONSE, - SERVER, - SWITCHED_PROTOCOL, -) +from .._state import CLIENT, IDLE, SEND_RESPONSE, SERVER from .._util import LocalProtocolError from .._writers import ( ChunkedWriter, @@ -61,7 +49,7 @@ ), ( (SERVER, SEND_RESPONSE), - Response(status_code=200, headers=[], reason=b"OK"), # type: ignore[arg-type] + Response(status_code=200, headers=[], reason=b"OK"), b"HTTP/1.1 200 OK\r\n\r\n", ), ( @@ -73,7 +61,7 @@ ), ( (SERVER, SEND_RESPONSE), - InformationalResponse(status_code=101, headers=[], reason=b"Upgrade"), # type: ignore[arg-type] + InformationalResponse(status_code=101, headers=[], reason=b"Upgrade"), b"HTTP/1.1 101 Upgrade\r\n\r\n", ), ] @@ -125,12 +113,12 @@ def check(got: Any) -> None: def test_writers_simple() -> None: - for ((role, state), event, binary) in SIMPLE_CASES: + for (role, state), event, binary in SIMPLE_CASES: tw(WRITERS[role, state], event, binary) def test_readers_simple() -> None: - for ((role, state), event, binary) in SIMPLE_CASES: + for (role, state), event, binary in SIMPLE_CASES: tr(READERS[role, state], binary, event) @@ -182,7 +170,7 @@ def test_readers_unusual() -> None: tr( READERS[CLIENT, IDLE], b"HEAD /foo HTTP/1.0\r\n\r\n", - Request(method="HEAD", target="/foo", headers=[], http_version="1.0"), # type: ignore[arg-type] + Request(method="HEAD", target="/foo", headers=[], http_version="1.0"), ) tr( @@ -364,18 +352,30 @@ def t_body_reader(thunk: Any, data: bytes, expected: Any, do_eof: bool = False) # Simple: consume whole thing print("Test 1") buf = makebuf(data) - assert _run_reader(thunk(), buf, do_eof) == expected + try: + assert _run_reader(thunk(), buf, do_eof) == expected + except LocalProtocolError: + if LocalProtocolError in expected: + pass + else: + raise # Incrementally growing buffer print("Test 2") reader = thunk() buf = ReceiveBuffer() events = [] - for i in range(len(data)): - events += _run_reader(reader, buf, False) - buf += data[i : i + 1] - events += _run_reader(reader, buf, do_eof) - assert normalize_data_events(events) == expected + try: + for i in range(len(data)): + events += _run_reader(reader, buf, False) + buf += data[i : i + 1] + events += _run_reader(reader, buf, do_eof) + assert normalize_data_events(events) == expected + except LocalProtocolError: + if LocalProtocolError in expected: + pass + else: + raise is_complete = any(type(event) is EndOfMessage for event in expected) if is_complete and not do_eof: @@ -436,14 +436,12 @@ def test_ChunkedReader() -> None: ) # refuses arbitrarily long chunk integers - with pytest.raises(LocalProtocolError): - # Technically this is legal HTTP/1.1, but we refuse to process chunk - # sizes that don't fit into 20 characters of hex - t_body_reader(ChunkedReader, b"9" * 100 + b"\r\nxxx", [Data(data=b"xxx")]) + # Technically this is legal HTTP/1.1, but we refuse to process chunk + # sizes that don't fit into 20 characters of hex + t_body_reader(ChunkedReader, b"9" * 100 + b"\r\nxxx", [LocalProtocolError]) # refuses garbage in the chunk count - with pytest.raises(LocalProtocolError): - t_body_reader(ChunkedReader, b"10\x00\r\nxxx", None) + t_body_reader(ChunkedReader, b"10\x00\r\nxxx", [LocalProtocolError]) # handles (and discards) "chunk extensions" omg wtf t_body_reader( @@ -457,10 +455,23 @@ def test_ChunkedReader() -> None: t_body_reader( ChunkedReader, - b"5 \r\n01234\r\n" + b"0\r\n\r\n", + b"5 \t \r\n01234\r\n" + b"0\r\n\r\n", [Data(data=b"01234"), EndOfMessage()], ) + # Chunked encoding with bad chunk termination characters are refused. Originally we + # simply dropped the 2 bytes after a chunk, instead of validating that the bytes + # were \r\n -- so we would successfully decode the data below as b"xxxa". And + # apparently there are other HTTP processors that ignore the chunk length and just + # keep reading until they see \r\n, so they would decode it as b"xxx__1a". Any time + # two HTTP processors accept the same input but interpret it differently, there's a + # possibility of request smuggling shenanigans. So we now reject this. + t_body_reader(ChunkedReader, b"3\r\nxxx__1a\r\n", [LocalProtocolError]) + + # Confirm we check both bytes individually + t_body_reader(ChunkedReader, b"3\r\nxxx\r_1a\r\n", [LocalProtocolError]) + t_body_reader(ChunkedReader, b"3\r\nxxx_\n1a\r\n", [LocalProtocolError]) + def test_ContentLengthWriter() -> None: w = ContentLengthWriter(5) @@ -483,8 +494,8 @@ def test_ContentLengthWriter() -> None: dowrite(w, EndOfMessage()) w = ContentLengthWriter(5) - dowrite(w, Data(data=b"123")) == b"123" - dowrite(w, Data(data=b"45")) == b"45" + assert dowrite(w, Data(data=b"123")) == b"123" + assert dowrite(w, Data(data=b"45")) == b"45" with pytest.raises(LocalProtocolError): dowrite(w, EndOfMessage(headers=[("Etag", "asdf")])) diff --git a/pyproject.toml b/pyproject.toml index edd11ae..64a6883 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,3 +38,9 @@ showcontent = true directory = "misc" name = "Miscellaneous internal changes" showcontent = true + +[tool.mypy] +strict = true +warn_unused_configs = true +warn_unused_ignores = true +show_error_codes = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 85dcc1f..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[mypy] -strict = true -warn_unused_configs = true -warn_unused_ignores = true -show_error_codes = true diff --git a/setup.py b/setup.py index 76db443..73713e2 100644 --- a/setup.py +++ b/setup.py @@ -12,16 +12,10 @@ author="Nathaniel J. Smith", author_email="njs@pobox.com", license="MIT", - packages=find_packages(), + packages=find_packages(exclude=["h11.tests"]), package_data={'h11': ['py.typed']}, url="https://github.com/python-hyper/h11", - # This means, just install *everything* you see under h11/, even if it - # doesn't look like a source file, so long as it appears in MANIFEST.in: - include_package_data=True, - python_requires=">=3.7", - install_requires=[ - "typing_extensions; python_version < '3.8'", - ], + python_requires=">=3.8", classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", @@ -30,10 +24,11 @@ "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Internet :: WWW/HTTP", "Topic :: System :: Networking", ], diff --git a/tox.ini b/tox.ini index 840b34c..6614ecf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,24 +1,24 @@ [tox] -envlist = format, py37, py38, py39, py310, pypy3, mypy +envlist = format, py{38, 39, 310, 311, 312, py3}, mypy [gh-actions] python = - 3.7: py37 - 3.8: py38 + 3.8: py38, format, mypy 3.9: py39 - 3.10: py310, format, mypy - pypy-3.7: pypy3 - pypy-3.8: pypy3 + 3.10: py310 + 3.11: py311 + 3.12: py312 + 3.13: py313 + pypy-3.9: pypy3 + pypy-3.10: pypy3 [testenv] deps = -r{toxinidir}/test-requirements.txt commands = pytest --cov=h11 --cov-config=.coveragerc h11 [testenv:format] -basepython = python3.10 -deps = - black - isort +basepython = python3.8 +deps = -r{toxinidir}/format-requirements.txt commands = black --check --diff h11/ bench/ examples/ fuzz/ isort --check --diff --profile black --dt h11 bench examples fuzz @@ -26,7 +26,7 @@ commands = [testenv:mypy] basepython = python3.8 deps = - mypy + mypy==1.8.0 pytest commands = mypy h11/