-
Notifications
You must be signed in to change notification settings - Fork 4.3k
/
Copy pathcreatedefaultroles.py
319 lines (268 loc) · 13.7 KB
/
createdefaultroles.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import logging
import re
import botocore.exceptions
import botocore.session
from botocore import xform_name
from awscli.customizations.utils import get_policy_arn_suffix
from awscli.customizations.emr import configutils
from awscli.customizations.emr import emrutils
from awscli.customizations.emr import exceptions
from awscli.customizations.emr.command import Command
from awscli.customizations.emr.constants import EC2
from awscli.customizations.emr.constants import EC2_ROLE_NAME
from awscli.customizations.emr.constants import EC2_SERVICE_PRINCIPAL
from awscli.customizations.emr.constants import ROLE_ARN_PATTERN
from awscli.customizations.emr.constants import EMR
from awscli.customizations.emr.constants import EMR_ROLE_NAME
from awscli.customizations.emr.constants import EMR_AUTOSCALING_ROLE_NAME
from awscli.customizations.emr.constants import APPLICATION_AUTOSCALING
from awscli.customizations.emr.constants import EC2_ROLE_POLICY_NAME
from awscli.customizations.emr.constants import EMR_ROLE_POLICY_NAME
from awscli.customizations.emr.constants \
import EMR_AUTOSCALING_ROLE_POLICY_NAME
from awscli.customizations.emr.constants import EMR_AUTOSCALING_SERVICE_NAME
from awscli.customizations.emr.constants \
import EMR_AUTOSCALING_SERVICE_PRINCIPAL
from awscli.customizations.emr.exceptions import ResolveServicePrincipalError
LOG = logging.getLogger(__name__)
def assume_role_policy(serviceprincipal):
return {
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {"Service": serviceprincipal},
"Action": "sts:AssumeRole"
}
]
}
def get_role_policy_arn(region, policy_name):
region_suffix = get_policy_arn_suffix(region)
role_arn = ROLE_ARN_PATTERN.replace("{{region_suffix}}", region_suffix)
role_arn = role_arn.replace("{{policy_name}}", policy_name)
return role_arn
def get_service_principal(service, endpoint_host, session=None):
if service == EC2:
return EC2_SERVICE_PRINCIPAL
suffix, region = _get_suffix_and_region_from_endpoint_host(endpoint_host)
if session is None:
session = botocore.session.Session()
if service == EMR_AUTOSCALING_SERVICE_NAME:
if region not in session.get_available_regions('emr', 'aws-cn'):
return EMR_AUTOSCALING_SERVICE_PRINCIPAL
return service + '.' + suffix
def _get_suffix_and_region_from_endpoint_host(endpoint_host):
suffix_match = _get_regex_match_from_endpoint_host(endpoint_host)
if suffix_match is not None and suffix_match.lastindex >= 3:
suffix = suffix_match.group(3)
region = suffix_match.group(2)
else:
raise ResolveServicePrincipalError
return suffix, region
def _get_regex_match_from_endpoint_host(endpoint_host):
if endpoint_host is None:
return None
regex_match = re.match("(https?://)([^.]+).elasticmapreduce.([^/]*)",
endpoint_host)
# Supports 'elasticmapreduce.{region}.' and '{region}.elasticmapreduce.'
if regex_match is None:
regex_match = re.match("(https?://elasticmapreduce).([^.]+).([^/]*)",
endpoint_host)
return regex_match
class CreateDefaultRoles(Command):
NAME = "create-default-roles"
DESCRIPTION = ('Creates the default IAM role ' +
EC2_ROLE_NAME + ' and ' +
EMR_ROLE_NAME + ' which can be used when creating the'
' cluster using the create-cluster command. The default'
' roles for EMR use managed policies, which are updated'
' automatically to support future EMR functionality.\n'
'\nIf you do not have a Service Role and Instance Profile '
'variable set for your create-cluster command in the AWS '
'CLI config file, create-default-roles will automatically '
'set the values for these variables with these default '
'roles. If you have already set a value for Service Role '
'or Instance Profile, create-default-roles will not '
'automatically set the defaults for these variables in the '
'AWS CLI config file. You can view settings for variables '
'in the config file using the "aws configure get" command.'
'\n')
ARG_TABLE = [
{'name': 'iam-endpoint',
'no_paramfile': True,
'help_text': '<p>The IAM endpoint to call for creating the roles.'
' This is optional and should only be specified when a'
' custom endpoint should be called for IAM operations'
'.</p>'}
]
def _run_main_command(self, parsed_args, parsed_globals):
self.iam_endpoint_url = parsed_args.iam_endpoint
self._check_for_iam_endpoint(self.region, self.iam_endpoint_url)
self.emr_endpoint_url = \
self._session.create_client(
'emr',
region_name=self.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl).meta.endpoint_url
LOG.debug('elasticmapreduce endpoint used for resolving'
' service principal: ' + self.emr_endpoint_url)
# Create default EC2 Role for EMR if it does not exist.
ec2_result, ec2_policy = self._create_role_if_not_exists(parsed_globals, EC2_ROLE_NAME,
EC2_ROLE_POLICY_NAME, [EC2])
# Create default EC2 Instance Profile for EMR if it does not exist.
instance_profile_name = EC2_ROLE_NAME
if self.check_if_instance_profile_exists(instance_profile_name,
parsed_globals):
LOG.debug('Instance Profile ' + instance_profile_name + ' exists.')
else:
LOG.debug('Instance Profile ' + instance_profile_name +
'does not exist. Creating default Instance Profile ' +
instance_profile_name)
self._create_instance_profile_with_role(instance_profile_name,
instance_profile_name,
parsed_globals)
# Create default EMR Role if it does not exist.
emr_result, emr_policy = self._create_role_if_not_exists(parsed_globals, EMR_ROLE_NAME,
EMR_ROLE_POLICY_NAME, [EMR])
# Create default EMR AutoScaling Role if it does not exist.
emr_autoscaling_result, emr_autoscaling_policy = \
self._create_role_if_not_exists(parsed_globals, EMR_AUTOSCALING_ROLE_NAME,
EMR_AUTOSCALING_ROLE_POLICY_NAME, [EMR, APPLICATION_AUTOSCALING])
configutils.update_roles(self._session)
emrutils.display_response(
self._session,
'create_role',
self._construct_result(ec2_result, ec2_policy,
emr_result, emr_policy,
emr_autoscaling_result, emr_autoscaling_policy),
parsed_globals)
return 0
def _create_role_if_not_exists(self, parsed_globals, role_name, policy_name, service_names):
result = None
policy = None
if self.check_if_role_exists(role_name, parsed_globals):
LOG.debug('Role ' + role_name + ' exists.')
else:
LOG.debug('Role ' + role_name + ' does not exist.'
' Creating default role: ' + role_name)
role_arn = get_role_policy_arn(self.region, policy_name)
result = self._create_role_with_role_policy(
role_name, service_names, role_arn, parsed_globals)
policy = self._get_role_policy(role_arn, parsed_globals)
return result, policy
def _check_for_iam_endpoint(self, region, iam_endpoint):
try:
self._session.create_client('emr', region)
except botocore.exceptions.UnknownEndpointError:
if iam_endpoint is None:
raise exceptions.UnknownIamEndpointError(region=region)
def _construct_result(self, ec2_response, ec2_policy,
emr_response, emr_policy,
emr_autoscaling_response, emr_autoscaling_policy):
result = []
self._construct_role_and_role_policy_structure(
result, ec2_response, ec2_policy)
self._construct_role_and_role_policy_structure(
result, emr_response, emr_policy)
self._construct_role_and_role_policy_structure(
result, emr_autoscaling_response, emr_autoscaling_policy)
return result
def _construct_role_and_role_policy_structure(
self, list, response, policy):
if response is not None and response['Role'] is not None:
list.append({'Role': response['Role'], 'RolePolicy': policy})
return list
def check_if_role_exists(self, role_name, parsed_globals):
parameters = {'RoleName': role_name}
try:
self._call_iam_operation('GetRole', parameters, parsed_globals)
except botocore.exceptions.ClientError as e:
role_not_found_code = "NoSuchEntity"
error_code = e.response.get('Error', {}).get('Code', '')
if role_not_found_code == error_code:
# No role error.
return False
else:
# Some other error. raise.
raise e
return True
def check_if_instance_profile_exists(self, instance_profile_name,
parsed_globals):
parameters = {'InstanceProfileName': instance_profile_name}
try:
self._call_iam_operation('GetInstanceProfile', parameters,
parsed_globals)
except botocore.exceptions.ClientError as e:
profile_not_found_code = 'NoSuchEntity'
error_code = e.response.get('Error', {}).get('Code')
if profile_not_found_code == error_code:
# No instance profile error.
return False
else:
# Some other error. raise.
raise e
return True
def _get_role_policy(self, arn, parsed_globals):
parameters = {}
parameters['PolicyArn'] = arn
policy_details = self._call_iam_operation('GetPolicy', parameters,
parsed_globals)
parameters["VersionId"] = policy_details["Policy"]["DefaultVersionId"]
policy_version_details = self._call_iam_operation('GetPolicyVersion',
parameters,
parsed_globals)
return policy_version_details["PolicyVersion"]["Document"]
def _create_role_with_role_policy(
self, role_name, service_names, role_arn, parsed_globals):
if len(service_names) == 1:
service_principal = get_service_principal(
service_names[0], self.emr_endpoint_url, self._session)
else:
service_principal = []
for service in service_names:
service_principal.append(get_service_principal(
service, self.emr_endpoint_url, self._session))
LOG.debug(f'Adding service principal(s) to trust policy: {service_principal}')
parameters = {'RoleName': role_name}
_assume_role_policy = \
emrutils.dict_to_string(assume_role_policy(service_principal))
parameters['AssumeRolePolicyDocument'] = _assume_role_policy
create_role_response = self._call_iam_operation('CreateRole',
parameters,
parsed_globals)
parameters = {}
parameters['PolicyArn'] = role_arn
parameters['RoleName'] = role_name
self._call_iam_operation('AttachRolePolicy',
parameters, parsed_globals)
return create_role_response
def _create_instance_profile_with_role(self, instance_profile_name,
role_name, parsed_globals):
# Creating an Instance Profile
parameters = {'InstanceProfileName': instance_profile_name}
self._call_iam_operation('CreateInstanceProfile', parameters,
parsed_globals)
# Adding the role to the Instance Profile
parameters = {}
parameters['InstanceProfileName'] = instance_profile_name
parameters['RoleName'] = role_name
self._call_iam_operation('AddRoleToInstanceProfile', parameters,
parsed_globals)
def _call_iam_operation(self, operation_name, parameters, parsed_globals):
client = self._session.create_client(
'iam', region_name=self.region, endpoint_url=self.iam_endpoint_url,
verify=parsed_globals.verify_ssl)
return getattr(client, xform_name(operation_name))(**parameters)