From 57bd1f02ca421fc57a724754da747843255f4028 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Tue, 26 Nov 2024 14:33:37 +0100 Subject: [PATCH 1/3] Apply IAM patches also on startup of STS, prevent patching multiple times --- .../localstack/services/iam/iam_patches.py | 122 +++++++++++++++ .../localstack/services/iam/provider.py | 139 +----------------- .../localstack/services/sts/provider.py | 4 + 3 files changed, 129 insertions(+), 136 deletions(-) create mode 100644 localstack-core/localstack/services/iam/iam_patches.py diff --git a/localstack-core/localstack/services/iam/iam_patches.py b/localstack-core/localstack/services/iam/iam_patches.py new file mode 100644 index 0000000000000..a41206a2095e2 --- /dev/null +++ b/localstack-core/localstack/services/iam/iam_patches.py @@ -0,0 +1,122 @@ +import threading +from typing import Optional + +from moto.iam.models import ( + AccessKey, + AWSManagedPolicy, + IAMBackend, + InlinePolicy, + Policy, +) +from moto.iam.models import Role as MotoRole +from moto.iam.policy_validation import VALID_STATEMENT_ELEMENTS + +from localstack import config +from localstack.utils.patch import patch + +ADDITIONAL_MANAGED_POLICIES = { + "AWSLambdaExecute": { + "Arn": "arn:aws:iam::aws:policy/AWSLambdaExecute", + "Path": "/", + "CreateDate": "2017-10-20T17:23:10+00:00", + "DefaultVersionId": "v4", + "Document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["logs:*"], + "Resource": "arn:aws:logs:*:*:*", + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": "arn:aws:s3:::*", + }, + ], + }, + "UpdateDate": "2019-05-20T18:22:18+00:00", + } +} + +IAM_PATCHED = False +IAM_PATCH_LOCK = threading.RLock() + + +def apply_iam_patches(): + global IAM_PATCHED + + # prevent patching multiple times, as this is called from both STS and IAM (for now) + with IAM_PATCH_LOCK: + if IAM_PATCHED: + return + + IAM_PATCHED = True + + # support service linked roles + moto_role_og_arn_prop = MotoRole.arn + + @property + def moto_role_arn(self): + return getattr(self, "service_linked_role_arn", None) or moto_role_og_arn_prop.__get__( + self + ) + + MotoRole.arn = moto_role_arn + + # Add missing managed polices + # TODO this might not be necessary + @patch(IAMBackend._init_aws_policies) + def _init_aws_policies_extended(_init_aws_policies, self): + loaded_policies = _init_aws_policies(self) + loaded_policies.extend( + [ + AWSManagedPolicy.from_data(name, self.account_id, self.region_name, d) + for name, d in ADDITIONAL_MANAGED_POLICIES.items() + ] + ) + return loaded_policies + + if "Principal" not in VALID_STATEMENT_ELEMENTS: + VALID_STATEMENT_ELEMENTS.append("Principal") + + # patch policy __init__ to set document as attribute + + @patch(Policy.__init__) + def policy__init__( + fn, + self, + name, + account_id, + region, + default_version_id=None, + description=None, + document=None, + **kwargs, + ): + fn(self, name, account_id, region, default_version_id, description, document, **kwargs) + self.document = document + + # patch unapply_policy + + @patch(InlinePolicy.unapply_policy) + def inline_policy_unapply_policy(fn, self, backend): + try: + fn(self, backend) + except Exception: + # Actually role can be deleted before policy being deleted in cloudformation + pass + + @patch(AccessKey.__init__) + def access_key__init__( + fn, + self, + user_name: Optional[str], + prefix: str, + account_id: str, + status: str = "Active", + **kwargs, + ): + if not config.PARITY_AWS_ACCESS_KEY_ID: + prefix = "L" + prefix[1:] + fn(self, user_name, prefix, account_id, status, **kwargs) diff --git a/localstack-core/localstack/services/iam/provider.py b/localstack-core/localstack/services/iam/provider.py index a4858cc4f0b41..7adca335e82da 100644 --- a/localstack-core/localstack/services/iam/provider.py +++ b/localstack-core/localstack/services/iam/provider.py @@ -1,22 +1,16 @@ import json import re from datetime import datetime -from typing import Dict, List, Optional +from typing import Dict, List from urllib.parse import quote from moto.iam.models import ( - AccessKey, - AWSManagedPolicy, IAMBackend, - InlinePolicy, - Policy, filter_items_with_path_prefix, iam_backends, ) from moto.iam.models import Role as MotoRole -from moto.iam.policy_validation import VALID_STATEMENT_ELEMENTS -from localstack import config from localstack.aws.api import CommonServiceException, RequestContext, handler from localstack.aws.api.iam import ( ActionNameListType, @@ -66,37 +60,13 @@ ) from localstack.aws.connect import connect_to from localstack.constants import INTERNAL_AWS_SECRET_ACCESS_KEY +from localstack.services.iam.iam_patches import apply_iam_patches from localstack.services.moto import call_moto from localstack.utils.aws.request_context import extract_access_key_id_from_auth_header from localstack.utils.common import short_uid -from localstack.utils.patch import patch SERVICE_LINKED_ROLE_PATH_PREFIX = "/aws-service-role" -ADDITIONAL_MANAGED_POLICIES = { - "AWSLambdaExecute": { - "Arn": "arn:aws:iam::aws:policy/AWSLambdaExecute", - "Path": "/", - "CreateDate": "2017-10-20T17:23:10+00:00", - "DefaultVersionId": "v4", - "Document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["logs:*"], - "Resource": "arn:aws:logs:*:*:*", - }, - { - "Effect": "Allow", - "Action": ["s3:GetObject", "s3:PutObject"], - "Resource": "arn:aws:s3:::*", - }, - ], - }, - "UpdateDate": "2019-05-20T18:22:18+00:00", - } -} POLICY_ARN_REGEX = re.compile(r"arn:[^:]+:iam::(?:\d{12}|aws):policy/.*") @@ -107,7 +77,7 @@ def get_iam_backend(context: RequestContext) -> IAMBackend: class IamProvider(IamApi): def __init__(self): - apply_patches() + apply_iam_patches() @handler("CreateRole", expand=False) def create_role( @@ -450,106 +420,3 @@ def attach_user_policy( if not POLICY_ARN_REGEX.match(policy_arn): raise InvalidInputException(f"ARN {policy_arn} is not valid.") return call_moto(context=context) - - # def get_user( - # self, context: RequestContext, user_name: existingUserNameType = None - # ) -> GetUserResponse: - # # TODO: The following migrates patch 'iam_response_get_user' as a provider function. - # # However, there are concerns with utilising 'aws_stack.extract_access_key_id_from_auth_header' - # # in place of 'moto.core.responses.get_current_user'. - # if not user_name: - # access_key_id = aws_stack.extract_access_key_id_from_auth_header(context.request.headers) - # moto_user = moto_iam_backend.get_user_from_access_key_id(access_key_id) - # if moto_user is None: - # moto_user = MotoUser("default_user") - # else: - # moto_user = moto_iam_backend.get_user(user_name) - # - # response_user_name = config.TEST_IAM_USER_NAME or moto_user.name - # response_user_id = config.TEST_IAM_USER_ID or moto_user.id - # moto_user = moto_iam_backend.users.get(response_user_name) or moto_user - # moto_tags = moto_iam_backend.tagger.list_tags_for_resource(moto_user.arn).get("Tags", []) - # response_tags = None - # if moto_tags: - # response_tags = [Tag(Key=t["Key"], Value=t["Value"]) for t in moto_tags] - # - # response_user = User() - # response_user["Path"] = moto_user.path - # response_user["UserName"] = response_user_name - # response_user["UserId"] = response_user_id - # response_user["Arn"] = moto_user.arn - # response_user["CreateDate"] = moto_user.create_date - # if moto_user.password_last_used: - # response_user["PasswordLastUsed"] = moto_user.password_last_used - # # response_user["PermissionsBoundary"] = # TODO - # if response_tags: - # response_user["Tags"] = response_tags - # return GetUserResponse(User=response_user) - - -def apply_patches(): - # support service linked roles - - @property - def moto_role_arn(self): - return getattr(self, "service_linked_role_arn", None) or moto_role_og_arn_prop.__get__(self) - - moto_role_og_arn_prop = MotoRole.arn - MotoRole.arn = moto_role_arn - - # Add missing managed polices - # TODO this might not be necessary - @patch(IAMBackend._init_aws_policies) - def _init_aws_policies_extended(_init_aws_policies, self): - loaded_policies = _init_aws_policies(self) - loaded_policies.extend( - [ - AWSManagedPolicy.from_data(name, self.account_id, self.region_name, d) - for name, d in ADDITIONAL_MANAGED_POLICIES.items() - ] - ) - return loaded_policies - - if "Principal" not in VALID_STATEMENT_ELEMENTS: - VALID_STATEMENT_ELEMENTS.append("Principal") - - # patch policy __init__ to set document as attribute - - @patch(Policy.__init__) - def policy__init__( - fn, - self, - name, - account_id, - region, - default_version_id=None, - description=None, - document=None, - **kwargs, - ): - fn(self, name, account_id, region, default_version_id, description, document, **kwargs) - self.document = document - - # patch unapply_policy - - @patch(InlinePolicy.unapply_policy) - def inline_policy_unapply_policy(fn, self, backend): - try: - fn(self, backend) - except Exception: - # Actually role can be deleted before policy being deleted in cloudformation - pass - - @patch(AccessKey.__init__) - def access_key__init__( - fn, - self, - user_name: Optional[str], - prefix: str, - account_id: str, - status: str = "Active", - **kwargs, - ): - if not config.PARITY_AWS_ACCESS_KEY_ID: - prefix = "L" + prefix[1:] - fn(self, user_name, prefix, account_id, status, **kwargs) diff --git a/localstack-core/localstack/services/sts/provider.py b/localstack-core/localstack/services/sts/provider.py index 90dad64269a77..006a510a612ce 100644 --- a/localstack-core/localstack/services/sts/provider.py +++ b/localstack-core/localstack/services/sts/provider.py @@ -18,6 +18,7 @@ tokenCodeType, unrestrictedSessionPolicyDocumentType, ) +from localstack.services.iam.iam_patches import apply_iam_patches from localstack.services.moto import call_moto from localstack.services.plugins import ServiceLifecycleHook from localstack.services.sts.models import sts_stores @@ -27,6 +28,9 @@ class StsProvider(StsApi, ServiceLifecycleHook): + def __init__(self): + apply_iam_patches() + def get_caller_identity(self, context: RequestContext, **kwargs) -> GetCallerIdentityResponse: response = call_moto(context) if "user/moto" in response["Arn"] and "sts" in response["Arn"]: From 96b5f3e64d1690f0c60d156aa42ef5d8b74f8e08 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Tue, 26 Nov 2024 14:44:52 +0100 Subject: [PATCH 2/3] Avoid patching blocking other provider loads for too long --- .../localstack/services/iam/iam_patches.py | 132 +++++++++--------- 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/localstack-core/localstack/services/iam/iam_patches.py b/localstack-core/localstack/services/iam/iam_patches.py index a41206a2095e2..5e5c9f5f4448f 100644 --- a/localstack-core/localstack/services/iam/iam_patches.py +++ b/localstack-core/localstack/services/iam/iam_patches.py @@ -53,70 +53,68 @@ def apply_iam_patches(): IAM_PATCHED = True - # support service linked roles - moto_role_og_arn_prop = MotoRole.arn - - @property - def moto_role_arn(self): - return getattr(self, "service_linked_role_arn", None) or moto_role_og_arn_prop.__get__( - self - ) - - MotoRole.arn = moto_role_arn - - # Add missing managed polices - # TODO this might not be necessary - @patch(IAMBackend._init_aws_policies) - def _init_aws_policies_extended(_init_aws_policies, self): - loaded_policies = _init_aws_policies(self) - loaded_policies.extend( - [ - AWSManagedPolicy.from_data(name, self.account_id, self.region_name, d) - for name, d in ADDITIONAL_MANAGED_POLICIES.items() - ] - ) - return loaded_policies - - if "Principal" not in VALID_STATEMENT_ELEMENTS: - VALID_STATEMENT_ELEMENTS.append("Principal") - - # patch policy __init__ to set document as attribute - - @patch(Policy.__init__) - def policy__init__( - fn, - self, - name, - account_id, - region, - default_version_id=None, - description=None, - document=None, - **kwargs, - ): - fn(self, name, account_id, region, default_version_id, description, document, **kwargs) - self.document = document - - # patch unapply_policy - - @patch(InlinePolicy.unapply_policy) - def inline_policy_unapply_policy(fn, self, backend): - try: - fn(self, backend) - except Exception: - # Actually role can be deleted before policy being deleted in cloudformation - pass - - @patch(AccessKey.__init__) - def access_key__init__( - fn, - self, - user_name: Optional[str], - prefix: str, - account_id: str, - status: str = "Active", - **kwargs, - ): - if not config.PARITY_AWS_ACCESS_KEY_ID: - prefix = "L" + prefix[1:] - fn(self, user_name, prefix, account_id, status, **kwargs) + # support service linked roles + moto_role_og_arn_prop = MotoRole.arn + + @property + def moto_role_arn(self): + return getattr(self, "service_linked_role_arn", None) or moto_role_og_arn_prop.__get__(self) + + MotoRole.arn = moto_role_arn + + # Add missing managed polices + # TODO this might not be necessary + @patch(IAMBackend._init_aws_policies) + def _init_aws_policies_extended(_init_aws_policies, self): + loaded_policies = _init_aws_policies(self) + loaded_policies.extend( + [ + AWSManagedPolicy.from_data(name, self.account_id, self.region_name, d) + for name, d in ADDITIONAL_MANAGED_POLICIES.items() + ] + ) + return loaded_policies + + if "Principal" not in VALID_STATEMENT_ELEMENTS: + VALID_STATEMENT_ELEMENTS.append("Principal") + + # patch policy __init__ to set document as attribute + + @patch(Policy.__init__) + def policy__init__( + fn, + self, + name, + account_id, + region, + default_version_id=None, + description=None, + document=None, + **kwargs, + ): + fn(self, name, account_id, region, default_version_id, description, document, **kwargs) + self.document = document + + # patch unapply_policy + + @patch(InlinePolicy.unapply_policy) + def inline_policy_unapply_policy(fn, self, backend): + try: + fn(self, backend) + except Exception: + # Actually role can be deleted before policy being deleted in cloudformation + pass + + @patch(AccessKey.__init__) + def access_key__init__( + fn, + self, + user_name: Optional[str], + prefix: str, + account_id: str, + status: str = "Active", + **kwargs, + ): + if not config.PARITY_AWS_ACCESS_KEY_ID: + prefix = "L" + prefix[1:] + fn(self, user_name, prefix, account_id, status, **kwargs) From fbcb484085ed9b6753621830e98dccc800bb2af8 Mon Sep 17 00:00:00 2001 From: Daniel Fangl Date: Tue, 26 Nov 2024 16:02:31 +0100 Subject: [PATCH 3/3] Fix test import --- tests/aws/services/iam/test_iam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/aws/services/iam/test_iam.py b/tests/aws/services/iam/test_iam.py index cf9913f0de98c..9f1bc02844f7c 100755 --- a/tests/aws/services/iam/test_iam.py +++ b/tests/aws/services/iam/test_iam.py @@ -5,7 +5,7 @@ from botocore.exceptions import ClientError from localstack.aws.api.iam import Tag -from localstack.services.iam.provider import ADDITIONAL_MANAGED_POLICIES +from localstack.services.iam.iam_patches import ADDITIONAL_MANAGED_POLICIES from localstack.testing.aws.util import create_client_with_keys, wait_for_user from localstack.testing.pytest import markers from localstack.utils.aws.arns import get_partition