Skip to content

Commit 97bc2c7

Browse files
dfangldfangl
authored
Add new volume dir mount option, rename VolumeBind to BindMount (#12471)
1 parent 7f920e3 commit 97bc2c7

File tree

10 files changed

+123
-44
lines changed

10 files changed

+123
-44
lines changed

‎localstack-core/localstack/dev/run/configurators.py

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010
from localstack import config, constants
1111
from localstack.utils.bootstrap import ContainerConfigurators
1212
from localstack.utils.container_utils.container_client import (
13+
BindMount,
1314
ContainerClient,
1415
ContainerConfiguration,
15-
VolumeBind,
1616
VolumeMappings,
1717
)
1818
from localstack.utils.docker_utils import DOCKER_CLIENT
@@ -107,7 +107,7 @@ def __call__(self, cfg: ContainerConfiguration):
107107
# encoding needs to be "utf-8" since scripts could include emojis
108108
file.write_text(self.script, newline="\n", encoding="utf-8")
109109
file.chmod(0o777)
110-
cfg.volumes.add(VolumeBind(str(file), f"/tmp/{file.name}"))
110+
cfg.volumes.add(BindMount(str(file), f"/tmp/{file.name}"))
111111
cfg.entrypoint = f"/tmp/{file.name}"
112112

113113

@@ -137,15 +137,15 @@ def __call__(self, cfg: ContainerConfiguration):
137137
cfg.volumes.add(
138138
# read_only=False is a temporary workaround to make the mounting of the pro source work
139139
# this can be reverted once we don't need the nested mounting anymore
140-
VolumeBind(str(source), self.container_paths.localstack_source_dir, read_only=False)
140+
BindMount(str(source), self.container_paths.localstack_source_dir, read_only=False)
141141
)
142142

143143
# ext source code if available
144144
if self.pro:
145145
source = self.host_paths.aws_pro_package_dir
146146
if source.exists():
147147
cfg.volumes.add(
148-
VolumeBind(
148+
BindMount(
149149
str(source), self.container_paths.localstack_pro_source_dir, read_only=True
150150
)
151151
)
@@ -163,7 +163,7 @@ def __call__(self, cfg: ContainerConfiguration):
163163
source = self.host_paths.localstack_project_dir / "bin" / "docker-entrypoint.sh"
164164
if source.exists():
165165
cfg.volumes.add(
166-
VolumeBind(str(source), self.container_paths.docker_entrypoint, read_only=True)
166+
BindMount(str(source), self.container_paths.docker_entrypoint, read_only=True)
167167
)
168168

169169
def try_mount_to_site_packages(self, cfg: ContainerConfiguration, sources_path: Path):
@@ -177,7 +177,7 @@ def try_mount_to_site_packages(self, cfg: ContainerConfiguration, sources_path:
177177
"""
178178
if sources_path.exists():
179179
cfg.volumes.add(
180-
VolumeBind(
180+
BindMount(
181181
str(sources_path),
182182
self.container_paths.dependency_source(sources_path.name),
183183
read_only=True,
@@ -219,7 +219,7 @@ def __call__(self, cfg: ContainerConfiguration):
219219
host_path = self.host_paths.aws_community_package_dir
220220
if host_path.exists():
221221
cfg.volumes.append(
222-
VolumeBind(
222+
BindMount(
223223
str(host_path), self.localstack_community_entry_points, read_only=True
224224
)
225225
)
@@ -244,7 +244,7 @@ def __call__(self, cfg: ContainerConfiguration):
244244
)
245245
if host_path.is_file():
246246
cfg.volumes.add(
247-
VolumeBind(
247+
BindMount(
248248
str(host_path),
249249
str(container_path),
250250
read_only=True,
@@ -260,7 +260,7 @@ def __call__(self, cfg: ContainerConfiguration):
260260
)
261261
if host_path.is_file():
262262
cfg.volumes.add(
263-
VolumeBind(
263+
BindMount(
264264
str(host_path),
265265
str(container_path),
266266
read_only=True,
@@ -270,7 +270,7 @@ def __call__(self, cfg: ContainerConfiguration):
270270
for host_path in self.host_paths.workspace_dir.glob(
271271
f"*/{dep}.egg-info/entry_points.txt"
272272
):
273-
cfg.volumes.add(VolumeBind(str(host_path), str(container_path), read_only=True))
273+
cfg.volumes.add(BindMount(str(host_path), str(container_path), read_only=True))
274274
break
275275

276276

@@ -330,7 +330,7 @@ def __call__(self, cfg: ContainerConfiguration):
330330
if self._has_mount(cfg.volumes, target_path):
331331
continue
332332

333-
cfg.volumes.append(VolumeBind(str(dep_path), target_path))
333+
cfg.volumes.append(BindMount(str(dep_path), target_path))
334334

335335
def _can_be_source_path(self, path: Path) -> bool:
336336
return path.is_dir() or (path.name.endswith(".py") and not path.name.startswith("__"))

‎localstack-core/localstack/services/lambda_/invocation/docker_runtime_executor.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,13 @@
3232
from localstack.services.lambda_.runtimes import IMAGE_MAPPING
3333
from localstack.utils.container_networking import get_main_container_name
3434
from localstack.utils.container_utils.container_client import (
35+
BindMount,
3536
ContainerConfiguration,
3637
DockerNotAvailable,
3738
DockerPlatform,
3839
NoSuchContainer,
3940
NoSuchImage,
4041
PortMappings,
41-
VolumeBind,
4242
VolumeMappings,
4343
)
4444
from localstack.utils.docker_utils import DOCKER_CLIENT as CONTAINER_CLIENT
@@ -331,7 +331,7 @@ def start(self, env_vars: dict[str, str]) -> None:
331331
if container_config.volumes is None:
332332
container_config.volumes = VolumeMappings()
333333
container_config.volumes.add(
334-
VolumeBind(
334+
BindMount(
335335
str(self.function_version.config.code.get_unzipped_code_location()),
336336
"/var/task",
337337
read_only=True,

‎localstack-core/localstack/utils/analytics/metadata.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -237,11 +237,11 @@ def prepare_host_machine_id():
237237

238238
@hooks.configure_localstack_container()
239239
def _mount_machine_file(container: Container):
240-
from localstack.utils.container_utils.container_client import VolumeBind
240+
from localstack.utils.container_utils.container_client import BindMount
241241

242242
# mount tha machine file from the host's CLI cache directory into the appropriate location in the
243243
# container
244244
machine_file = os.path.join(config.dirs.cache, "machine.json")
245245
if os.path.isfile(machine_file):
246246
target = os.path.join(config.dirs.for_container().cache, "machine.json")
247-
container.config.volumes.add(VolumeBind(machine_file, target, read_only=True))
247+
container.config.volumes.add(BindMount(machine_file, target, read_only=True))

‎localstack-core/localstack/utils/bootstrap.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from localstack.runtime import hooks
2525
from localstack.utils.container_networking import get_main_container_name
2626
from localstack.utils.container_utils.container_client import (
27+
BindMount,
2728
CancellableStream,
2829
ContainerClient,
2930
ContainerConfiguration,
@@ -33,7 +34,7 @@
3334
NoSuchImage,
3435
NoSuchNetwork,
3536
PortMappings,
36-
VolumeBind,
37+
VolumeDirMount,
3738
VolumeMappings,
3839
)
3940
from localstack.utils.container_utils.docker_cmd_client import CmdDockerClient
@@ -491,7 +492,7 @@ def mount_docker_socket(cfg: ContainerConfiguration):
491492
target = "/var/run/docker.sock"
492493
if cfg.volumes.find_target_mapping(target):
493494
return
494-
cfg.volumes.add(VolumeBind(source, target))
495+
cfg.volumes.add(BindMount(source, target))
495496
cfg.env_vars["DOCKER_HOST"] = f"unix://{target}"
496497

497498
@staticmethod
@@ -501,7 +502,7 @@ def mount_localstack_volume(host_path: str | os.PathLike = None):
501502
def _cfg(cfg: ContainerConfiguration):
502503
if cfg.volumes.find_target_mapping(constants.DEFAULT_VOLUME_DIR):
503504
return
504-
cfg.volumes.add(VolumeBind(str(host_path), constants.DEFAULT_VOLUME_DIR))
505+
cfg.volumes.add(BindMount(str(host_path), constants.DEFAULT_VOLUME_DIR))
505506

506507
return _cfg
507508

@@ -679,7 +680,7 @@ def _cfg(cfg: ContainerConfiguration):
679680
return _cfg
680681

681682
@staticmethod
682-
def volume(volume: VolumeBind):
683+
def volume(volume: BindMount | VolumeDirMount):
683684
def _cfg(cfg: ContainerConfiguration):
684685
cfg.volumes.add(volume)
685686

@@ -807,7 +808,7 @@ def volume_cli_params(params: Iterable[str] = None):
807808

808809
def _cfg(cfg: ContainerConfiguration):
809810
for param in params:
810-
cfg.volumes.append(VolumeBind.parse(param))
811+
cfg.volumes.append(BindMount.parse(param))
811812

812813
return _cfg
813814

‎localstack-core/localstack/utils/container_utils/container_client.py

Lines changed: 62 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import dotenv
2828

2929
from localstack import config
30+
from localstack.constants import DEFAULT_VOLUME_DIR
3031
from localstack.utils.collections import HashableList, ensure_list
3132
from localstack.utils.files import TMP_FILES, chmod_r, rm_rf, save_file
3233
from localstack.utils.no_exit_argument_parser import NoExitArgumentParser
@@ -370,7 +371,7 @@ def __repr__(self):
370371

371372

372373
@dataclasses.dataclass
373-
class VolumeBind:
374+
class BindMount:
374375
"""Represents a --volume argument run/create command. When using VolumeBind to bind-mount a file or directory
375376
that does not yet exist on the Docker host, -v creates the endpoint for you. It is always created as a directory.
376377
"""
@@ -395,8 +396,14 @@ def to_str(self) -> str:
395396

396397
return ":".join(args)
397398

399+
def to_docker_sdk_parameters(self) -> tuple[str, dict[str, str]]:
400+
return str(self.host_dir), {
401+
"bind": self.container_dir,
402+
"mode": "ro" if self.read_only else "rw",
403+
}
404+
398405
@classmethod
399-
def parse(cls, param: str) -> "VolumeBind":
406+
def parse(cls, param: str) -> "BindMount":
400407
parts = param.split(":")
401408
if 1 > len(parts) > 3:
402409
raise ValueError(f"Cannot parse volume bind {param}")
@@ -408,27 +415,66 @@ def parse(cls, param: str) -> "VolumeBind":
408415
return volume
409416

410417

418+
@dataclasses.dataclass
419+
class VolumeDirMount:
420+
volume_path: str
421+
"""
422+
Absolute path inside /var/lib/localstack to mount into the container
423+
"""
424+
container_path: str
425+
"""
426+
Target path inside the started container
427+
"""
428+
read_only: bool = False
429+
430+
def to_str(self) -> str:
431+
self._validate()
432+
from localstack.utils.docker_utils import get_host_path_for_path_in_docker
433+
434+
host_dir = get_host_path_for_path_in_docker(self.volume_path)
435+
return f"{host_dir}:{self.container_path}{':ro' if self.read_only else ''}"
436+
437+
def _validate(self):
438+
if not self.volume_path:
439+
raise ValueError("no volume dir specified")
440+
if config.is_in_docker and not self.volume_path.startswith(DEFAULT_VOLUME_DIR):
441+
raise ValueError(f"volume dir not starting with {DEFAULT_VOLUME_DIR}")
442+
if not self.container_path:
443+
raise ValueError("no container dir specified")
444+
445+
def to_docker_sdk_parameters(self) -> tuple[str, dict[str, str]]:
446+
self._validate()
447+
from localstack.utils.docker_utils import get_host_path_for_path_in_docker
448+
449+
host_dir = get_host_path_for_path_in_docker(self.volume_path)
450+
return host_dir, {
451+
"bind": self.container_path,
452+
"mode": "ro" if self.read_only else "rw",
453+
}
454+
455+
411456
class VolumeMappings:
412-
mappings: List[Union[SimpleVolumeBind, VolumeBind]]
457+
mappings: List[Union[SimpleVolumeBind, BindMount]]
413458

414-
def __init__(self, mappings: List[Union[SimpleVolumeBind, VolumeBind]] = None):
459+
def __init__(self, mappings: List[Union[SimpleVolumeBind, BindMount, VolumeDirMount]] = None):
415460
self.mappings = mappings if mappings is not None else []
416461

417-
def add(self, mapping: Union[SimpleVolumeBind, VolumeBind]):
462+
def add(self, mapping: Union[SimpleVolumeBind, BindMount, VolumeDirMount]):
418463
self.append(mapping)
419464

420465
def append(
421466
self,
422467
mapping: Union[
423468
SimpleVolumeBind,
424-
VolumeBind,
469+
BindMount,
470+
VolumeDirMount,
425471
],
426472
):
427473
self.mappings.append(mapping)
428474

429475
def find_target_mapping(
430476
self, container_dir: str
431-
) -> Optional[Union[SimpleVolumeBind, VolumeBind]]:
477+
) -> Optional[Union[SimpleVolumeBind, BindMount, VolumeDirMount]]:
432478
"""
433479
Looks through the volumes and returns the one where the container dir matches ``container_dir``.
434480
Returns None if there is no volume mapping to the given container directory.
@@ -448,6 +494,12 @@ def __iter__(self):
448494
def __repr__(self):
449495
return self.mappings.__repr__()
450496

497+
def __len__(self):
498+
return len(self.mappings)
499+
500+
def __getitem__(self, item: int):
501+
return self.mappings[item]
502+
451503

452504
VolumeType = Literal["bind", "volume"]
453505

@@ -1441,12 +1493,9 @@ def convert_mount_list_to_dict(
14411493
) -> Dict[str, Dict[str, str]]:
14421494
"""Converts a List of (host_path, container_path) tuples to a Dict suitable as volume argument for docker sdk"""
14431495

1444-
def _map_to_dict(paths: SimpleVolumeBind | VolumeBind):
1445-
if isinstance(paths, VolumeBind):
1446-
return str(paths.host_dir), {
1447-
"bind": paths.container_dir,
1448-
"mode": "ro" if paths.read_only else "rw",
1449-
}
1496+
def _map_to_dict(paths: SimpleVolumeBind | BindMount | VolumeDirMount):
1497+
if isinstance(paths, (BindMount, VolumeDirMount)):
1498+
return paths.to_docker_sdk_parameters()
14501499
else:
14511500
return str(paths[0]), {"bind": paths[1], "mode": "rw"}
14521501

‎localstack-core/localstack/utils/container_utils/docker_cmd_client.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from localstack.utils.collections import ensure_list
1313
from localstack.utils.container_utils.container_client import (
1414
AccessDenied,
15+
BindMount,
1516
CancellableStream,
1617
ContainerClient,
1718
ContainerException,
@@ -29,7 +30,7 @@
2930
SimpleVolumeBind,
3031
Ulimit,
3132
Util,
32-
VolumeBind,
33+
VolumeDirMount,
3334
)
3435
from localstack.utils.run import run
3536
from localstack.utils.strings import first_char_to_upper, to_str
@@ -878,7 +879,7 @@ def _build_run_create_cmd(
878879
return cmd, env_file
879880

880881
@staticmethod
881-
def _map_to_volume_param(volume: Union[SimpleVolumeBind, VolumeBind]) -> str:
882+
def _map_to_volume_param(volume: Union[SimpleVolumeBind, BindMount, VolumeDirMount]) -> str:
882883
"""
883884
Maps the mount volume, to a parameter for the -v docker cli argument.
884885
@@ -889,7 +890,7 @@ def _map_to_volume_param(volume: Union[SimpleVolumeBind, VolumeBind]) -> str:
889890
:param volume: Either a SimpleVolumeBind, in essence a tuple (host_dir, container_dir), or a VolumeBind object
890891
:return: String which is passable as parameter to the docker cli -v option
891892
"""
892-
if isinstance(volume, VolumeBind):
893+
if isinstance(volume, (BindMount, VolumeDirMount)):
893894
return volume.to_str()
894895
else:
895896
return f"{volume[0]}:{volume[1]}"

‎tests/bootstrap/test_container_configurators.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
get_gateway_url,
1010
)
1111
from localstack.utils.common import external_service_ports
12-
from localstack.utils.container_utils.container_client import VolumeBind
12+
from localstack.utils.container_utils.container_client import BindMount
1313

1414

1515
def test_common_container_fixture_configurators(
@@ -96,7 +96,7 @@ def test_custom_command_configurator(container_factory, tmp_path, stream_contain
9696
ContainerConfigurators.custom_command(
9797
["/tmp/pytest-tmp-path/my-command.sh", "hello", "world"]
9898
),
99-
ContainerConfigurators.volume(VolumeBind(str(tmp_path), "/tmp/pytest-tmp-path")),
99+
ContainerConfigurators.volume(BindMount(str(tmp_path), "/tmp/pytest-tmp-path")),
100100
],
101101
remove=False,
102102
)

‎tests/bootstrap/test_init.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from localstack.config import in_docker
77
from localstack.testing.pytest.container import ContainerFactory
88
from localstack.utils.bootstrap import ContainerConfigurators
9-
from localstack.utils.container_utils.container_client import VolumeBind
9+
from localstack.utils.container_utils.container_client import BindMount
1010

1111
pytestmarks = pytest.mark.skipif(
1212
condition=in_docker(), reason="cannot run bootstrap tests in docker"
@@ -43,7 +43,7 @@ def test_shutdown_hooks(
4343
ContainerConfigurators.default_gateway_port,
4444
ContainerConfigurators.mount_localstack_volume(volume),
4545
ContainerConfigurators.volume(
46-
VolumeBind(str(shutdown_hooks), "/etc/localstack/init/shutdown.d")
46+
BindMount(str(shutdown_hooks), "/etc/localstack/init/shutdown.d")
4747
),
4848
]
4949
)

0 commit comments

Comments
 (0)