fix logic to proxy S3 requests against *amazonaws.com (#83)

Author: whummer

aws-replicator/Makefile

@@ -33,7 +33,7 @@ install: venv
 	$(VENV_RUN); $(PIP_CMD) install -e ".[test]"
 
 test: venv
-	$(VENV_RUN); python -m pytest $(TEST_PATH)
+	$(VENV_RUN); python -m pytest $(PYTEST_ARGS) $(TEST_PATH)
 
 dist: venv
 	$(VENV_RUN); python setup.py sdist bdist_wheel

aws-replicator/README.md

@@ -126,6 +126,7 @@ If you wish to access the deprecated instructions, they can be found [here](http
 
 ## Change Log
 
+* `0.1.20`: Fix logic for proxying S3 requests with `*.s3.amazonaws.com` host header
 * `0.1.19`: Print human-readable message for invalid regexes in resource configs; fix logic for proxying S3 requests with host-based addressing
 * `0.1.18`: Update environment check to use SDK Docker client and enable starting the proxy from within Docker (e.g., from the LS main container as part of an init script)
 * `0.1.17`: Add basic support for ARN-based pattern-matching for `secretsmanager` resources

aws-replicator/aws_replicator/client/auth_proxy.py

@@ -275,9 +275,10 @@ def _fix_headers(self, request: Request, service_name: str):
     def _fix_host_and_path(self, request: Request, service_name: str):
         if service_name == "s3":
             # fix the path and prepend the bucket name, to avoid bucket addressing issues
+            regex_base_domain = rf"((amazonaws\.com)|({LOCALHOST_HOSTNAME}))"
             host = request.headers.pop(HEADER_HOST_ORIGINAL, None)
             host = host or request.headers.get("Host") or ""
-            match = re.match(rf"(.+)\.s3\.{LOCALHOST_HOSTNAME}", host)
+            match = re.match(rf"(.+)\.s3\.{regex_base_domain}", host)
             if match:
                 # prepend the bucket name (extracted from the host) to the path of the request (path-based addressing)
                 request.path = f"/{match.group(1)}{request.path}"

aws-replicator/setup.cfg

@@ -1,8 +1,8 @@
 [metadata]
 name = localstack-extension-aws-replicator
-version = 0.1.19
-summary = LocalStack Extension: AWS replicator
-description = Replicate AWS resources into your LocalStack instance
+version = 0.1.20
+summary = LocalStack AWS Proxy Extension
+description = Proxy AWS resources into your LocalStack instance
 long_description = file: README.md
 long_description_content_type = text/markdown; charset=UTF-8
 url = https://github.com/localstack/localstack-extensions/tree/main/aws-replicator

aws-replicator/tests/test_proxy_requests.py

@@ -1,6 +1,8 @@
 # Note: these tests depend on the extension being installed and actual AWS credentials being configured, such
 # that the proxy can be started within the tests. They are designed to be mostly run in CI at this point.
 import gzip
+import re
+from urllib.parse import urlparse
 
 import boto3
 import pytest
@@ -41,20 +43,33 @@ def _start(config: dict = None):
 
 
 @pytest.mark.parametrize("metadata_gzip", [True, False])
-@pytest.mark.parametrize("host_addressing", [True, False])
-def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip, host_addressing):
+@pytest.mark.parametrize("target_endpoint", ["local_domain", "aws_domain", "default"])
+def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip, target_endpoint):
     # start proxy
     config = ProxyConfig(services={"s3": {"resources": ".*"}}, bind_host=PROXY_BIND_HOST)
     start_aws_proxy(config)
 
     # create clients
-    if host_addressing:
+    if target_endpoint == "default":
+        s3_client = connect_to().s3
+    else:
         s3_client = connect_to(
             endpoint_url="http://s3.localhost.localstack.cloud:4566",
             config=Config(s3={"addressing_style": "virtual"}),
         ).s3
-    else:
-        s3_client = connect_to().s3
+
+    if target_endpoint == "aws_domain":
+
+        def _add_header(request, **kwargs):
+            # instrument boto3 client to add custom `Host` header, mimicking a `*.s3.amazonaws.com` request
+            url = urlparse(request.url)
+            match = re.match(r"(.+)\.s3\.localhost\.localstack\.cloud", url.netloc)
+            if match:
+                request.headers.add_header("host", f"{match.group(1)}.s3.amazonaws.com")
+
+        s3_client.meta.events.register_first("before-sign.*.*", _add_header)
+
+    # define S3 client pointing to real AWS
     s3_client_aws = boto3.client("s3")
 
     # list buckets to assert that proxy is up and running