Skip to content

Fix kubernetes dev run script #12229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 6, 2025
Merged

Fix kubernetes dev run script #12229

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a5e752f
First changes to kubernetes dev run
dfangl Feb 3, 2025
bffb86a
Refacotr kubernetes dev run script and adapt it for namespace packagi…
dfangl Feb 4, 2025
a7be55e
Add some useful environment variables per default if pro is used
dfangl Feb 4, 2025
10c0590
Remove testing line, print warning if mount-entrypoint is used
dfangl Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
243 changes: 162 additions & 81 deletions localstack-core/localstack/dev/kubernetes/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,56 +1,152 @@
import dataclasses
import os
from typing import Literal

import click
import yaml

from localstack import version as localstack_version

@dataclasses.dataclass
class MountPoint:
name: str
host_path: str
container_path: str
node_path: str
read_only: bool = True
volume_type: Literal["Directory", "File"] = "Directory"

def generate_k8s_cluster_config(pro: bool = False, mount_moto: bool = False, port: int = 4566):
volumes = []

def generate_mount_points(
pro: bool = False, mount_moto: bool = False, mount_entrypoints: bool = False
) -> list[MountPoint]:
mount_points = []
# host paths
root_path = os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")
localstack_code_path = os.path.join(root_path, "localstack-core", "localstack")
volumes.append(
{
"volume": f"{os.path.normpath(localstack_code_path)}:/code/localstack",
"nodeFilters": ["server:*", "agent:*"],
}
)
pro_path = os.path.join(root_path, "..", "localstack-ext")

egg_path = os.path.join(
root_path, "localstack-core", "localstack_core.egg-info/entry_points.txt"
)
volumes.append(
{
"volume": f"{os.path.normpath(egg_path)}:/code/entry_points_community",
"nodeFilters": ["server:*", "agent:*"],
}
)
# container paths
target_path = "/opt/code/localstack/"
venv_path = os.path.join(target_path, ".venv", "lib", "python3.11", "site-packages")

# Community code
if pro:
pro_path = os.path.join(root_path, "..", "localstack-ext")
pro_code_path = os.path.join(pro_path, "localstack-pro-core", "localstack", "pro", "core")
volumes.append(
{
"volume": f"{os.path.normpath(pro_code_path)}:/code/localstack_ext",
"nodeFilters": ["server:*", "agent:*"],
}
# Pro installs community code as a package, so it lives in the venv site-packages
mount_points.append(
MountPoint(
name="localstack",
host_path=os.path.normpath(localstack_code_path),
node_path="/code/localstack",
container_path=os.path.join(venv_path, "localstack"),
# Read only has to be false here, as we mount the pro code into this mount, as it is the entire namespace package
read_only=False,
)
)

egg_path = os.path.join(
pro_path, "localstack-pro-core", "localstack_ext.egg-info/entry_points.txt"
else:
# Community does not install the localstack package in the venv, but has the code directly in `/opt/code/localstack`
mount_points.append(
MountPoint(
name="localstack",
host_path=os.path.normpath(localstack_code_path),
node_path="/code/localstack",
container_path=os.path.join(target_path, "localstack-core", "localstack"),
)
)
volumes.append(
{
"volume": f"{os.path.normpath(egg_path)}:/code/entry_points_ext",
"nodeFilters": ["server:*", "agent:*"],
}

# Pro code
if pro:
pro_code_path = os.path.join(pro_path, "localstack-pro-core", "localstack", "pro", "core")
mount_points.append(
MountPoint(
name="localstack-pro",
host_path=os.path.normpath(pro_code_path),
node_path="/code/localstack-pro",
container_path=os.path.join(venv_path, "localstack", "pro", "core"),
)
)

# entrypoints
if mount_entrypoints:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we log a warning that this is currently not really implemented, and may provide confusing results if the user expects this to work

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, note for the future: we implement this in localstack.dev.run so we can take the implementation from there.

See localstack.dev.run.configurators._list_files_in_container_image

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will add some logging. The implementation can be used, the main issue is that the image in docker is not necessarily the same image in k3d, as it has a separate image store, and could pull the image (or just have a different version pulled). We would need to check the image version inside k3d, if the cluster is already running, or if not, we could check the latest one, but in general it is not guaranteed.

if pro:
# Community entrypoints in pro image
# TODO actual package version detection
print(
"WARNING: Package version detection is not implemented."
"You need to adapt the version in the .egg-info paths to match the package version installed in the used localstack-pro image."
)
community_version = "4.1.1.dev14"
pro_version = "4.1.1.dev16"
egg_path = os.path.join(
root_path, "localstack-core", "localstack_core.egg-info/entry_points.txt"
)
mount_points.append(
MountPoint(
name="entry-points-community",
host_path=os.path.normpath(egg_path),
node_path="/code/entry-points-community",
container_path=os.path.join(
venv_path, f"localstack-{community_version}.egg-info", "entry_points.txt"
),
volume_type="File",
)
)
# Pro entrypoints in pro image
egg_path = os.path.join(
pro_path, "localstack-pro-core", "localstack_ext.egg-info/entry_points.txt"
)
mount_points.append(
MountPoint(
name="entry-points-pro",
host_path=os.path.normpath(egg_path),
node_path="/code/entry-points-pro",
container_path=os.path.join(
venv_path, f"localstack_ext-{pro_version}.egg-info", "entry_points.txt"
),
volume_type="File",
)
)
else:
# Community entrypoints in community repo
# In the community image, the code is not installed as package, so the paths are predictable
egg_path = os.path.join(
root_path, "localstack-core", "localstack_core.egg-info/entry_points.txt"
)
mount_points.append(
MountPoint(
name="entry-points-community",
host_path=os.path.normpath(egg_path),
node_path="/code/entry-points-community",
container_path=os.path.join(
target_path,
"localstack-core",
"localstack_core.egg-info",
"entry_points.txt",
),
volume_type="File",
)
)

if mount_moto:
moto_path = os.path.join(root_path, "..", "moto", "moto")
volumes.append(
{"volume": f"{moto_path}:/code/moto", "nodeFilters": ["server:*", "agent:*"]}
mount_points.append(
MountPoint(
name="moto",
host_path=os.path.normpath(moto_path),
node_path="/code/moto",
container_path=os.path.join(venv_path, "moto"),
)
)
return mount_points


def generate_k8s_cluster_config(mount_points: list[MountPoint], port: int = 4566):
volumes = [
{
"volume": f"{mount_point.host_path}:{mount_point.node_path}",
"nodeFilters": ["server:*", "agent:*"],
}
for mount_point in mount_points
]

ports = [{"port": f"{port}:31566", "nodeFilters": ["server:0"]}]

Expand All @@ -64,49 +160,24 @@ def snake_to_kebab_case(string: str):


def generate_k8s_cluster_overrides(
pro: bool = False, cluster_config: dict = None, env: list[str] | None = None
mount_points: list[MountPoint], pro: bool = False, env: list[str] | None = None
):
volumes = []
for volume in cluster_config["volumes"]:
name = snake_to_kebab_case(volume["volume"].split(":")[-1].split("/")[-1])
volume_type = "Directory" if name != "entry-points" else "File"
volumes.append(
{
"name": name,
"hostPath": {"path": volume["volume"].split(":")[-1], "type": volume_type},
}
)

volume_mounts = []
target_path = "/opt/code/localstack/"
venv_path = os.path.join(target_path, ".venv", "lib", "python3.11", "site-packages")
for volume in volumes:
if volume["name"] == "entry-points":
entry_points_path = os.path.join(
target_path, "localstack_core.egg-info", "entry_points.txt"
)
if pro:
project = "localstack_ext-"
version = localstack_version.__version__
dist_info = f"{project}{version}0.dist-info"
entry_points_path = os.path.join(venv_path, dist_info, "entry_points.txt")

volume_mounts.append(
{
"name": volume["name"],
"readOnly": True,
"mountPath": entry_points_path,
}
)
continue
volumes = [
{
"name": mount_point.name,
"hostPath": {"path": mount_point.node_path, "type": mount_point.volume_type},
}
for mount_point in mount_points
]

volume_mounts.append(
{
"name": volume["name"],
"readOnly": True,
"mountPath": os.path.join(venv_path, volume["hostPath"]["path"].split("/")[-1]),
}
)
volume_mounts = [
{
"name": mount_point.name,
"readOnly": mount_point.read_only,
"mountPath": mount_point.container_path,
}
for mount_point in mount_points
]

extra_env_vars = []
if env:
Expand All @@ -120,12 +191,16 @@ def generate_k8s_cluster_overrides(
)

if pro:
extra_env_vars.append(
extra_env_vars += [
{
"name": "LOCALSTACK_AUTH_TOKEN",
"value": "test",
}
)
},
{
"name": "CONTAINER_RUNTIME",
"value": "kubernetes",
},
]

image_repository = "localstack/localstack-pro" if pro else "localstack/localstack"

Expand All @@ -135,6 +210,7 @@ def generate_k8s_cluster_overrides(
"volumeMounts": volume_mounts,
"extraEnvVars": extra_env_vars,
"image": {"repository": image_repository},
"lambda": {"executor": "kubernetes"},
}

return overrides
Expand Down Expand Up @@ -162,6 +238,9 @@ def print_file(content: dict, file_name: str):
@click.option(
"--mount-moto", is_flag=True, default=None, help="Mount the moto code into the cluster."
)
@click.option(
"--mount-entrypoints", is_flag=True, default=None, help="Mount the entrypoints into the pod."
)
@click.option(
"--write",
is_flag=True,
Expand Down Expand Up @@ -200,6 +279,7 @@ def print_file(content: dict, file_name: str):
def run(
pro: bool = None,
mount_moto: bool = False,
mount_entrypoints: bool = False,
write: bool = False,
output_dir=None,
overrides_file: str = None,
Expand All @@ -211,10 +291,11 @@ def run(
"""
A tool for localstack developers to generate the kubernetes cluster configuration file and the overrides to mount the localstack code into the cluster.
"""
mount_points = generate_mount_points(pro, mount_moto, mount_entrypoints)

config = generate_k8s_cluster_config(pro=pro, mount_moto=mount_moto, port=port)
config = generate_k8s_cluster_config(mount_points, port=port)

overrides = generate_k8s_cluster_overrides(pro, config, env=env)
overrides = generate_k8s_cluster_overrides(mount_points, pro=pro, env=env)

output_dir = output_dir or os.getcwd()
overrides_file = overrides_file or "overrides.yml"
Expand Down
Loading