Skip to content

Commit fe0c97e

Browse files
whummerwhummer
authored
catch regex config errors in AWS proxy; fix logic for proxying S3 requests with host-based addressing (#80)
1 parent 4cb621c commit fe0c97e

File tree

7 files changed

+63
-34
lines changed

7 files changed

+63
-34
lines changed

‎aws-replicator/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ enable: $(wildcard ./build/dist/localstack_extension_aws_replicator-*.tar.gz) #
4949
localstack extensions -v install file://$?
5050

5151
publish: clean-dist venv dist
52-
$(VENV_RUN); cd build; pip install --upgrade twine; twine upload dist/*
52+
$(VENV_RUN); pip install --upgrade twine; twine upload dist/*
5353

5454
clean-dist: clean
5555
rm -rf dist/

‎aws-replicator/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ localstack extensions install "git+https://github.com/localstack/localstack-exte
152152

153153
## Change Log
154154

155+
* `0.1.19`: Print human-readable message for invalid regexes in resource configs; fix logic for proxying S3 requests with host-based addressing
155156
* `0.1.18`: Update environment check to use SDK Docker client and enable starting the proxy from within Docker (e.g., from the LS main container as part of an init script)
156157
* `0.1.17`: Add basic support for ARN-based pattern-matching for `secretsmanager` resources
157158
* `0.1.16`: Update imports for localstack >=3.6 compatibility

‎aws-replicator/aws_replicator/client/auth_proxy.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from localstack import config as localstack_config
1616
from localstack.aws.spec import load_service
1717
from localstack.config import external_service_url
18-
from localstack.constants import AWS_REGION_US_EAST_1, DOCKER_IMAGE_NAME_PRO
18+
from localstack.constants import AWS_REGION_US_EAST_1, DOCKER_IMAGE_NAME_PRO, LOCALHOST_HOSTNAME
1919
from localstack.http import Request
2020
from localstack.utils.aws.aws_responses import requests_response
2121
from localstack.utils.bootstrap import setup_logging
@@ -32,6 +32,7 @@
3232
from aws_replicator import config as repl_config
3333
from aws_replicator.client.utils import truncate_content
3434
from aws_replicator.config import HANDLER_PATH_PROXIES
35+
from aws_replicator.shared.constants import HEADER_HOST_ORIGINAL
3536
from aws_replicator.shared.models import AddProxyRequest, ProxyConfig
3637

3738
from .http2_server import run_server
@@ -106,6 +107,7 @@ def proxy_request(self, request: Request, data: bytes) -> Response:
106107

107108
# fix headers (e.g., "Host") and create client
108109
self._fix_headers(request, service_name)
110+
self._fix_host_and_path(request, service_name)
109111

110112
# create request and request dict
111113
operation_model, aws_request, request_dict = self._parse_aws_request(
@@ -262,14 +264,24 @@ def _fix_headers(self, request: Request, service_name: str):
262264
host = request.headers.get("Host") or ""
263265
regex = r"^(https?://)?([0-9.]+|localhost)(:[0-9]+)?"
264266
if re.match(regex, host):
265-
request.headers["Host"] = re.sub(regex, r"\1s3.localhost.localstack.cloud", host)
267+
request.headers["Host"] = re.sub(regex, rf"\1s3.{LOCALHOST_HOSTNAME}", host)
266268
request.headers.pop("Content-Length", None)
267269
request.headers.pop("x-localstack-request-url", None)
268270
request.headers.pop("X-Forwarded-For", None)
269271
request.headers.pop("X-Localstack-Tgt-Api", None)
270272
request.headers.pop("X-Moto-Account-Id", None)
271273
request.headers.pop("Remote-Addr", None)
272274

275+
def _fix_host_and_path(self, request: Request, service_name: str):
276+
if service_name == "s3":
277+
# fix the path and prepend the bucket name, to avoid bucket addressing issues
278+
host = request.headers.pop(HEADER_HOST_ORIGINAL, None)
279+
host = host or request.headers.get("Host") or ""
280+
match = re.match(rf"(.+)\.s3\.{LOCALHOST_HOSTNAME}", host)
281+
if match:
282+
# prepend the bucket name (extracted from the host) to the path of the request (path-based addressing)
283+
request.path = f"/{match.group(1)}{request.path}"
284+
273285
def _extract_region_and_service(self, headers) -> Optional[Tuple[str, str]]:
274286
auth_header = headers.pop("Authorization", "")
275287
parts = auth_header.split("Credential=", maxsplit=1)

‎aws-replicator/aws_replicator/server/aws_request_forwarder.py

Lines changed: 34 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
except ImportError:
2323
from localstack.constants import TEST_AWS_ACCESS_KEY_ID
2424

25+
from aws_replicator.shared.constants import HEADER_HOST_ORIGINAL
2526
from aws_replicator.shared.models import ProxyInstance, ProxyServiceConfig
2627

2728
LOG = logging.getLogger(__name__)
@@ -98,33 +99,38 @@ def select_proxy(self, context: RequestContext) -> Optional[ProxyInstance]:
9899
def _request_matches_resource(
99100
self, context: RequestContext, resource_name_pattern: str
100101
) -> bool:
101-
service_name = self._get_canonical_service_name(context.service.service_name)
102-
if service_name == "s3":
103-
bucket_name = context.service_request.get("Bucket") or ""
104-
s3_bucket_arn = arns.s3_bucket_arn(bucket_name)
105-
return bool(re.match(resource_name_pattern, s3_bucket_arn))
106-
if service_name == "sqs":
107-
queue_name = context.service_request.get("QueueName") or ""
108-
queue_url = context.service_request.get("QueueUrl") or ""
109-
queue_name = queue_name or queue_url.split("/")[-1]
110-
candidates = (
111-
queue_name,
112-
queue_url,
113-
sqs_queue_arn(
114-
queue_name, account_id=context.account_id, region_name=context.region
115-
),
116-
)
117-
for candidate in candidates:
118-
if re.match(resource_name_pattern, candidate):
119-
return True
120-
return False
121-
if service_name == "secretsmanager":
122-
secret_id = context.service_request.get("SecretId") or ""
123-
secret_arn = secretsmanager_secret_arn(
124-
secret_id, account_id=context.account_id, region_name=context.region
125-
)
126-
return bool(re.match(resource_name_pattern, secret_arn))
127-
# TODO: add more resource patterns
102+
try:
103+
service_name = self._get_canonical_service_name(context.service.service_name)
104+
if service_name == "s3":
105+
bucket_name = context.service_request.get("Bucket") or ""
106+
s3_bucket_arn = arns.s3_bucket_arn(bucket_name)
107+
return bool(re.match(resource_name_pattern, s3_bucket_arn))
108+
if service_name == "sqs":
109+
queue_name = context.service_request.get("QueueName") or ""
110+
queue_url = context.service_request.get("QueueUrl") or ""
111+
queue_name = queue_name or queue_url.split("/")[-1]
112+
candidates = (
113+
queue_name,
114+
queue_url,
115+
sqs_queue_arn(
116+
queue_name, account_id=context.account_id, region_name=context.region
117+
),
118+
)
119+
for candidate in candidates:
120+
if re.match(resource_name_pattern, candidate):
121+
return True
122+
return False
123+
if service_name == "secretsmanager":
124+
secret_id = context.service_request.get("SecretId") or ""
125+
secret_arn = secretsmanager_secret_arn(
126+
secret_id, account_id=context.account_id, region_name=context.region
127+
)
128+
return bool(re.match(resource_name_pattern, secret_arn))
129+
# TODO: add more resource patterns
130+
except re.error as e:
131+
raise Exception(
132+
"Error evaluating regular expression - please verify proxy configuration"
133+
) from e
128134
return True
129135

130136
def forward_request(self, context: RequestContext, proxy: ProxyInstance) -> requests.Response:
@@ -140,7 +146,7 @@ def forward_request(self, context: RequestContext, proxy: ProxyInstance) -> requ
140146

141147
result = None
142148
try:
143-
headers.pop("Host", None)
149+
headers[HEADER_HOST_ORIGINAL] = headers.pop("Host", None)
144150
headers.pop("Content-Length", None)
145151
ctype = headers.get("Content-Type")
146152
data = b""

‎aws-replicator/aws_replicator/shared/constants.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# header name for the original request host name forwarded in the request to the target proxy handler
2+
HEADER_HOST_ORIGINAL = "x-ls-host-original"

‎aws-replicator/setup.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[metadata]
22
name = localstack-extension-aws-replicator
3-
version = 0.1.18
3+
version = 0.1.19
44
summary = LocalStack Extension: AWS replicator
55
description = Replicate AWS resources into your LocalStack instance
66
long_description = file: README.md

‎aws-replicator/tests/test_proxy_requests.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import boto3
66
import pytest
7+
from botocore.client import Config
78
from botocore.exceptions import ClientError
89
from localstack.aws.connect import connect_to
910
from localstack.utils.aws.arns import sqs_queue_arn, sqs_queue_url_for_arn
@@ -40,13 +41,20 @@ def _start(config: dict = None):
4041

4142

4243
@pytest.mark.parametrize("metadata_gzip", [True, False])
43-
def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip):
44+
@pytest.mark.parametrize("host_addressing", [True, False])
45+
def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip, host_addressing):
4446
# start proxy
4547
config = ProxyConfig(services={"s3": {"resources": ".*"}}, bind_host=PROXY_BIND_HOST)
4648
start_aws_proxy(config)
4749

4850
# create clients
49-
s3_client = connect_to().s3
51+
if host_addressing:
52+
s3_client = connect_to(
53+
endpoint_url="http://s3.localhost.localstack.cloud:4566",
54+
config=Config(s3={"addressing_style": "virtual"}),
55+
).s3
56+
else:
57+
s3_client = connect_to().s3
5058
s3_client_aws = boto3.client("s3")
5159

5260
# list buckets to assert that proxy is up and running

0 commit comments

Comments
 (0)