Skip to content

Commit 06b3e40

Browse files
bentskubentsku
committed
fix auto decoding of gzip for proxied requests
1 parent 1c32bf4 commit 06b3e40

File tree

3 files changed

+86
-10
lines changed

3 files changed

+86
-10
lines changed

‎localstack/http/client.py

Lines changed: 61 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,19 @@ class SimpleRequestsClient(HttpClient):
4747
def __init__(self, session: requests.Session = None):
4848
self.session = session or requests.Session()
4949

50+
@staticmethod
51+
def _get_destination_url(request: Request, server: str | None = None) -> str:
52+
if server:
53+
# accepts "http://localhost:5000" or "localhost:5000"
54+
if "://" in server:
55+
parts = urlparse(server)
56+
scheme, server = parts.scheme, parts.netloc
57+
else:
58+
scheme = request.scheme
59+
return get_raw_current_url(scheme, server, request.root_path, get_raw_path(request))
60+
61+
return get_raw_base_url(request)
62+
5063
def request(self, request: Request, server: str | None = None) -> Response:
5164
"""
5265
Very naive implementation to make the given HTTP request using the requests library, i.e., process the request
@@ -61,16 +74,7 @@ def request(self, request: Request, server: str | None = None) -> Response:
6174
:return: the response.
6275
"""
6376

64-
if server:
65-
# accepts "http://localhost:5000" or "localhost:5000"
66-
if "://" in server:
67-
parts = urlparse(server)
68-
scheme, server = parts.scheme, parts.netloc
69-
else:
70-
scheme = request.scheme
71-
url = get_raw_current_url(scheme, server, request.root_path, get_raw_path(request))
72-
else:
73-
url = get_raw_base_url(request)
77+
url = self._get_destination_url(request, server)
7478

7579
response = self.session.request(
7680
method=request.method,
@@ -97,6 +101,53 @@ def close(self):
97101
self.session.close()
98102

99103

104+
class SimpleStreamingRequestsClient(SimpleRequestsClient):
105+
def request(self, request: Request, server: str | None = None) -> Response:
106+
"""
107+
Very naive implementation to make the given HTTP request using the requests library, i.e., process the request
108+
as a client.
109+
110+
:param request: the request to perform
111+
:param server: the URL to send the request to, which defaults to the host component of the original Request.
112+
:return: the response.
113+
"""
114+
115+
url = self._get_destination_url(request, server)
116+
117+
response = self.session.request(
118+
method=request.method,
119+
# use raw base url to preserve path url encoding
120+
url=url,
121+
# request.args are only the url parameters
122+
params=[(k, v) for k, v in request.args.items(multi=True)],
123+
headers=dict(request.headers.items()),
124+
data=restore_payload(request),
125+
stream=True,
126+
)
127+
128+
if request.method == "HEAD":
129+
# for HEAD requests we have to keep the original content-length, but it will be re-calculated when creating
130+
# the final_response object
131+
final_response = Response(
132+
response=response.content,
133+
status=response.status_code,
134+
headers=Headers(dict(response.headers)),
135+
)
136+
final_response.content_length = response.headers.get("Content-Length", 0)
137+
return final_response
138+
139+
response_headers = Headers(dict(response.headers))
140+
response_headers.pop("Content-Length", None)
141+
142+
final_response = Response(
143+
response=(chunk for chunk in response.raw.stream(1024, decode_content=False)),
144+
status=response.status_code,
145+
headers=response_headers,
146+
)
147+
148+
return final_response
149+
150+
100151
def make_request(request: Request) -> Response:
101152
"""
102153
Convenience method to make the given HTTP as a client.

‎localstack/services/s3/virtual_host.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from localstack import config
66
from localstack.constants import LOCALHOST_HOSTNAME
77
from localstack.http import Request, Response
8+
from localstack.http.client import SimpleStreamingRequestsClient
89
from localstack.http.proxy import Proxy
910
from localstack.runtime import hooks
1011
from localstack.services.edge import ROUTER
@@ -45,6 +46,7 @@ def __call__(self, request: Request, **kwargs) -> Response:
4546
with Proxy(
4647
forward_base_url=config.get_edge_url(),
4748
preserve_host=False,
49+
client=SimpleStreamingRequestsClient(),
4850
) as proxy:
4951
forwarded = proxy.forward(
5052
request=request, forward_path=forward_to_url.path, headers=copied_headers

‎tests/integration/s3/test_s3.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1820,6 +1820,29 @@ def test_put_object_chunked_newlines(self, s3_bucket, aws_client):
18201820
assert len(body) == len(str(download_file_object))
18211821
assert body == str(download_file_object)
18221822

1823+
@pytest.mark.only_localstack
1824+
def test_virtual_host_proxy_does_not_decode_gzip(self, aws_client, s3_bucket):
1825+
# Write contents to memory rather than a file.
1826+
data = "123gzipfile"
1827+
upload_file_object = BytesIO()
1828+
mtime = 1676569620 # hardcode the GZIP timestamp
1829+
with gzip.GzipFile(fileobj=upload_file_object, mode="w", mtime=mtime) as filestream:
1830+
filestream.write(data.encode("utf-8"))
1831+
raw_gzip = upload_file_object.getvalue()
1832+
# Upload gzip
1833+
aws_client.s3.put_object(
1834+
Bucket=s3_bucket,
1835+
Key="test.gz",
1836+
ContentEncoding="gzip",
1837+
Body=raw_gzip,
1838+
)
1839+
1840+
key_url = f"{_bucket_url_vhost(s3_bucket)}/test.gz"
1841+
gzip_response = requests.get(key_url, stream=True)
1842+
# get the raw data, don't let requests decode the response
1843+
raw_data = b"".join(chunk for chunk in gzip_response.raw.stream(1024, decode_content=False))
1844+
assert raw_data == raw_gzip
1845+
18231846
@pytest.mark.only_localstack
18241847
def test_put_object_with_md5_and_chunk_signature(self, s3_bucket, aws_client):
18251848
# Boto still does not support chunk encoding, which means we can't test with the client nor

0 commit comments

Comments
 (0)