Skip to content

Commit 752e01d

Browse files
simonrwsimonrw
authored
CFn: partial implementation of language extensions transform (#12813)
1 parent e38daad commit 752e01d

14 files changed

+1342
-13
lines changed

‎localstack-core/localstack/services/cloudformation/engine/entities.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ def __init__(
104104
self.template_original = clone_safe(self.template)
105105
# initialize resources
106106
for resource_id, resource in self.template_resources.items():
107+
# HACK: if the resource is a Fn::ForEach intrinsic call from the LanguageExtensions transform, then it is not a dictionary but a list
108+
if resource_id.startswith("Fn::ForEach"):
109+
# we are operating on an untransformed template, so ignore for now
110+
continue
107111
resource["LogicalResourceId"] = self.template_original["Resources"][resource_id][
108112
"LogicalResourceId"
109113
] = resource.get("LogicalResourceId") or resource_id

‎localstack-core/localstack/services/cloudformation/engine/transformers.py

Lines changed: 180 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1+
import copy
12
import json
23
import logging
34
import os
5+
import re
46
from copy import deepcopy
5-
from typing import Dict, Optional, Type, Union
7+
from dataclasses import dataclass
8+
from typing import Any, Callable, Dict, Optional, Type, Union
69

710
import boto3
811
from botocore.exceptions import ClientError
@@ -12,6 +15,7 @@
1215
from localstack.aws.connect import connect_to
1316
from localstack.services.cloudformation.engine.policy_loader import create_policy_loader
1417
from localstack.services.cloudformation.engine.template_deployer import resolve_refs_recursively
18+
from localstack.services.cloudformation.engine.validations import ValidationError
1519
from localstack.services.cloudformation.stores import get_cloudformation_store
1620
from localstack.utils import testutil
1721
from localstack.utils.objects import recurse_object
@@ -26,6 +30,29 @@
2630
TransformResult = Union[dict, str]
2731

2832

33+
@dataclass
34+
class ResolveRefsRecursivelyContext:
35+
account_id: str
36+
region_name: str
37+
stack_name: str
38+
resources: dict
39+
mappings: dict
40+
conditions: dict
41+
parameters: dict
42+
43+
def resolve(self, value: Any) -> Any:
44+
return resolve_refs_recursively(
45+
self.account_id,
46+
self.region_name,
47+
self.stack_name,
48+
self.resources,
49+
self.mappings,
50+
self.conditions,
51+
self.parameters,
52+
value,
53+
)
54+
55+
2956
class Transformer:
3057
"""Abstract class for Fn::Transform intrinsic functions"""
3158

@@ -155,7 +182,20 @@ def apply_global_transformations(
155182
account_id, region_name, processed_template, stack_parameters
156183
)
157184
elif transformation["Name"] == EXTENSIONS_TRANSFORM:
158-
continue
185+
resolve_context = ResolveRefsRecursivelyContext(
186+
account_id,
187+
region_name,
188+
stack_name,
189+
resources,
190+
mappings,
191+
conditions,
192+
stack_parameters,
193+
)
194+
195+
processed_template = apply_language_extensions_transform(
196+
processed_template,
197+
resolve_context,
198+
)
159199
elif transformation["Name"] == SECRETSMANAGER_TRANSFORM:
160200
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/transform-aws-secretsmanager.html
161201
LOG.warning("%s is not yet supported. Ignoring.", SECRETSMANAGER_TRANSFORM)
@@ -269,6 +309,144 @@ def execute_macro(
269309
return result.get("fragment")
270310

271311

312+
def apply_language_extensions_transform(
313+
template: dict,
314+
resolve_context: ResolveRefsRecursivelyContext,
315+
) -> dict:
316+
"""
317+
Resolve language extensions constructs
318+
"""
319+
320+
def _visit(obj, path, **_):
321+
# Fn::ForEach
322+
# TODO: can this be used in non-resource positions?
323+
if isinstance(obj, dict) and any("Fn::ForEach" in key for key in obj):
324+
newobj = {}
325+
for key in obj:
326+
if "Fn::ForEach" not in key:
327+
newobj[key] = obj[key]
328+
continue
329+
330+
new_entries = expand_fn_foreach(obj[key], resolve_context)
331+
newobj.update(**new_entries)
332+
return newobj
333+
# Fn::Length
334+
elif isinstance(obj, dict) and "Fn::Length" in obj:
335+
value = obj["Fn::Length"]
336+
if isinstance(value, dict):
337+
value = resolve_context.resolve(value)
338+
339+
if isinstance(value, list):
340+
# TODO: what if one of the elements was AWS::NoValue?
341+
# no conversion required
342+
return len(value)
343+
elif isinstance(value, str):
344+
length = len(value.split(","))
345+
return length
346+
return obj
347+
elif isinstance(obj, dict) and "Fn::ToJsonString" in obj:
348+
# TODO: is the default representation ok here?
349+
return json.dumps(obj["Fn::ToJsonString"], default=str, separators=(",", ":"))
350+
351+
# reference
352+
return obj
353+
354+
return recurse_object(template, _visit)
355+
356+
357+
def expand_fn_foreach(
358+
foreach_defn: list,
359+
resolve_context: ResolveRefsRecursivelyContext,
360+
extra_replace_mapping: dict | None = None,
361+
) -> dict:
362+
if len(foreach_defn) != 3:
363+
raise ValidationError(
364+
f"Fn::ForEach: invalid number of arguments, expected 3 got {len(foreach_defn)}"
365+
)
366+
output = {}
367+
iteration_name, iteration_value, template = foreach_defn
368+
if not isinstance(iteration_name, str):
369+
raise ValidationError(
370+
f"Fn::ForEach: incorrect type for iteration name '{iteration_name}', expected str"
371+
)
372+
if isinstance(iteration_value, dict):
373+
# we have a reference
374+
if "Ref" in iteration_value:
375+
iteration_value = resolve_context.resolve(iteration_value)
376+
else:
377+
raise NotImplementedError(
378+
f"Fn::Transform: intrinsic {iteration_value} not supported in this position yet"
379+
)
380+
if not isinstance(iteration_value, list):
381+
raise ValidationError(
382+
f"Fn::ForEach: incorrect type for iteration variables '{iteration_value}', expected list"
383+
)
384+
385+
if not isinstance(template, dict):
386+
raise ValidationError(
387+
f"Fn::ForEach: incorrect type for template '{template}', expected dict"
388+
)
389+
390+
# TODO: locations other than resources
391+
replace_template_value = "${" + iteration_name + "}"
392+
for variable in iteration_value:
393+
# there might be multiple children, which could themselves be a `Fn::ForEach` call
394+
for logical_resource_id_template in template:
395+
if logical_resource_id_template.startswith("Fn::ForEach"):
396+
result = expand_fn_foreach(
397+
template[logical_resource_id_template],
398+
resolve_context,
399+
{iteration_name: variable},
400+
)
401+
output.update(**result)
402+
continue
403+
404+
if replace_template_value not in logical_resource_id_template:
405+
raise ValidationError("Fn::ForEach: no placeholder in logical resource id")
406+
407+
def gen_visit(variable: str) -> Callable:
408+
def _visit(obj: Any, path: Any):
409+
if isinstance(obj, dict) and "Ref" in obj:
410+
ref_variable = obj["Ref"]
411+
if ref_variable == iteration_name:
412+
return variable
413+
elif isinstance(obj, dict) and "Fn::Sub" in obj:
414+
arguments = recurse_object(obj["Fn::Sub"], _visit)
415+
if isinstance(arguments, str):
416+
# simple case
417+
# TODO: can this reference anything outside of the template?
418+
result = arguments
419+
variables_found = re.findall("\\${([^}]+)}", arguments)
420+
for var in variables_found:
421+
if var == iteration_name:
422+
result = result.replace(f"${{{var}}}", variable)
423+
return result
424+
else:
425+
raise NotImplementedError
426+
elif isinstance(obj, dict) and "Fn::Join" in obj:
427+
# first visit arguments
428+
arguments = recurse_object(
429+
obj["Fn::Join"],
430+
_visit,
431+
)
432+
separator, items = arguments
433+
return separator.join(items)
434+
return obj
435+
436+
return _visit
437+
438+
logical_resource_id = logical_resource_id_template.replace(
439+
replace_template_value, variable
440+
)
441+
for key, value in (extra_replace_mapping or {}).items():
442+
logical_resource_id = logical_resource_id.replace("${" + key + "}", value)
443+
resource_body = copy.deepcopy(template[logical_resource_id_template])
444+
body = recurse_object(resource_body, gen_visit(variable))
445+
output[logical_resource_id] = body
446+
447+
return output
448+
449+
272450
def apply_serverless_transformation(
273451
account_id: str, region_name: str, parsed_template: dict, template_parameters: dict
274452
) -> Optional[str]:

‎localstack-core/localstack/services/cloudformation/engine/v2/change_set_model_executor.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -355,12 +355,13 @@ def _execute_resource_action(
355355
)
356356
resource_provider = resource_provider_executor.try_load_resource_provider(resource_type)
357357
track_resource_operation(action, resource_type, missing=resource_provider is not None)
358-
log_not_available_message(
359-
resource_type,
360-
f'No resource provider found for "{resource_type}"',
361-
)
362-
if resource_provider is None and not config.CFN_IGNORE_UNSUPPORTED_RESOURCE_TYPES:
363-
raise NoResourceProvider
358+
if resource_provider is None:
359+
log_not_available_message(
360+
resource_type,
361+
f'No resource provider found for "{resource_type}"',
362+
)
363+
if not config.CFN_IGNORE_UNSUPPORTED_RESOURCE_TYPES:
364+
raise NoResourceProvider
364365

365366
extra_resource_properties = {}
366367
event = ProgressEvent(OperationStatus.SUCCESS, resource_model={})

‎localstack-core/localstack/services/cloudformation/provider.py

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,6 @@ def create_stack(self, context: RequestContext, request: CreateStackInput) -> Cr
244244
old_parameters={},
245245
)
246246

247-
# handle conditions
248247
stack = Stack(context.account_id, context.region, request, template)
249248

250249
try:
@@ -269,12 +268,15 @@ def create_stack(self, context: RequestContext, request: CreateStackInput) -> Cr
269268
state.stacks[stack.stack_id] = stack
270269
return CreateStackOutput(StackId=stack.stack_id)
271270

271+
# HACK: recreate the stack (including all of its confusing processes in the __init__ method
272+
# to set the stack template to be the transformed template, rather than the untransformed
273+
# template
274+
stack = Stack(context.account_id, context.region, request, template)
275+
272276
# perform basic static analysis on the template
273277
for validation_fn in DEFAULT_TEMPLATE_VALIDATIONS:
274278
validation_fn(template)
275279

276-
stack = Stack(context.account_id, context.region, request, template)
277-
278280
# resolve conditions
279281
raw_conditions = template.get("Conditions", {})
280282
resolved_stack_conditions = resolve_stack_conditions(
@@ -512,8 +514,18 @@ def get_template(
512514

513515
if template_stage == TemplateStage.Processed and "Transform" in stack.template_body:
514516
copy_template = clone(stack.template_original)
515-
copy_template.pop("ChangeSetName", None)
516-
copy_template.pop("StackName", None)
517+
for key in [
518+
"ChangeSetName",
519+
"StackName",
520+
"StackId",
521+
"Transform",
522+
"Conditions",
523+
"Mappings",
524+
]:
525+
copy_template.pop(key, None)
526+
for key in ["Parameters", "Outputs"]:
527+
if key in copy_template and not copy_template[key]:
528+
copy_template.pop(key)
517529
for resource in copy_template.get("Resources", {}).values():
518530
resource.pop("LogicalResourceId", None)
519531
template_body = json.dumps(copy_template)

‎localstack-core/localstack/testing/pytest/fixtures.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1106,6 +1106,8 @@ def _deploy(
11061106

11071107
if template_path is not None:
11081108
template = load_template_file(template_path)
1109+
if template is None:
1110+
raise RuntimeError(f"Could not find file {os.path.realpath(template_path)}")
11091111
template_rendered = render_template(template, **(template_mapping or {}))
11101112

11111113
kwargs = dict(

0 commit comments

Comments
 (0)