From 324fb8ebda3fc7de8b6de01dbc96a670c6e4cf06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=93=E6=BA=AA?= Date: Tue, 12 Aug 2025 04:52:15 +0800 Subject: [PATCH] build-large-scale-low-cost-real-time-log-management-platform --- .../README.md | 59 ++++ .../main.tf | 292 ++++++++++++++++++ .../outputs.tf | 14 + .../variables.tf | 22 ++ 4 files changed, 387 insertions(+) create mode 100644 solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/README.md create mode 100644 solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/main.tf create mode 100644 solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/outputs.tf create mode 100644 solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/variables.tf diff --git a/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/README.md b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/README.md new file mode 100644 index 0000000000..484656d582 --- /dev/null +++ b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/README.md @@ -0,0 +1,59 @@ +## Introduction + + +本示例用于实现解决方案[开源自建ELK上云指南：基于阿里云日志服务（SLS）构建低成本可扩展日志平台](https://www.aliyun.com/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform), 涉及到专有网络（VPC）、交换机（VSwitch）、云服务器（ECS）、RAM 用户等资源的创建。 + + + +This example demonstrates the implementation of the solution [Build Large Scale Low Cost Realtime Log Management Platform](https://www.aliyun.com/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform). It involves the creation, and deployment of resources such as Virtual Private Cloud (VPC), VSwitch, Elastic Compute Service (ECS), and RAM users. + + + +## Providers + +| Name | Version | +|------|---------| +| [alicloud](#provider\_alicloud) | n/a | +| [random](#provider\_random) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [alicloud_ecs_command.run_command](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_command) | resource | +| [alicloud_ecs_command.run_command_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_command) | resource | +| [alicloud_ecs_invocation.invoke_script](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_invocation) | resource | +| [alicloud_ecs_invocation.invoke_script_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_invocation) | resource | +| [alicloud_instance.ecs_instance](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/instance) | resource | +| [alicloud_instance.ecs_instance_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/instance) | resource | +| [alicloud_log_machine_group.this](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_machine_group) | resource | +| [alicloud_log_project.sls_project](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_project) | resource | +| [alicloud_log_store.sls_log_store](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_store) | resource | +| [alicloud_log_store_index.sls_index](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_store_index) | resource | +| [alicloud_logtail_attachment.this](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/logtail_attachment) | resource | +| [alicloud_logtail_config.this](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/logtail_config) | resource | +| [alicloud_ram_access_key.ramak](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ram_access_key) | resource | +| [alicloud_ram_user.ram_user](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ram_user) | resource | +| [alicloud_ram_user_policy_attachment.attach_policy_to_user](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ram_user_policy_attachment) | resource | +| [alicloud_security_group.security_group](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/security_group) | resource | +| [alicloud_security_group.security_group_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/security_group) | resource | +| [alicloud_security_group_rule.allow_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/security_group_rule) | resource | +| [alicloud_vpc.vpc](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/vpc) | resource | +| [alicloud_vswitch.vswitch](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/vswitch) | resource | +| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [alicloud_images.default](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/data-sources/images) | data source | +| [alicloud_zones.default](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/data-sources/zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ecs\_instance\_password](#input\_ecs\_instance\_password) | 服务器登录密码,长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）` | `string` | n/a | yes | +| [instance\_type](#input\_instance\_type) | 实例类型 | `string` | `"ecs.e-c1m2.large"` | no | +| [instance\_type\_xlarge](#input\_instance\_type\_xlarge) | 高性能实例类型 | `string` | `"ecs.e-c1m2.xlarge"` | no | +| [region](#input\_region) | n/a | `string` | `"cn-hangzhou"` | no | + \ No newline at end of file diff --git a/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/main.tf b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/main.tf new file mode 100644 index 0000000000..2de935ddb5 --- /dev/null +++ b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/main.tf @@ -0,0 +1,292 @@ +provider "alicloud" { + region = var.region +} + +data "alicloud_zones" "default" { + available_disk_category = "cloud_essd" + available_resource_creation = "VSwitch" + available_instance_type = var.instance_type +} + +resource "random_string" "suffix" { + length = 8 + lower = true + upper = false + numeric = false + special = false +} + +locals { + common_name = random_string.suffix.id +} + +resource "alicloud_vpc" "vpc" { + cidr_block = "192.168.0.0/16" + vpc_name = "vpc-${local.common_name}" +} + +resource "alicloud_vswitch" "vswitch" { + vpc_id = alicloud_vpc.vpc.id + cidr_block = "192.168.0.0/24" + zone_id = data.alicloud_zones.default.zones.0.id + vswitch_name = "vswitch-${local.common_name}" +} + +resource "alicloud_security_group" "security_group" { + vpc_id = alicloud_vpc.vpc.id + security_group_name = "sg-${local.common_name}" +} + +data "alicloud_images" "default" { + name_regex = "^aliyun_3_x64_20G_alibase_.*" + most_recent = true + owners = "system" +} + +resource "alicloud_ram_user" "ram_user" { + name = "create_by_solution-${local.common_name}" +} + +resource "alicloud_ram_access_key" "ramak" { + user_name = alicloud_ram_user.ram_user.name + depends_on = [ + alicloud_ram_user.ram_user + ] +} + +resource "alicloud_ram_user_policy_attachment" "attach_policy_to_user" { + user_name = alicloud_ram_user.ram_user.name + policy_type = "System" + policy_name = "AliyunLogFullAccess" + depends_on = [ + alicloud_ram_access_key.ramak + ] +} + +# the ECS instance which generate the log, and where LoongCollector is installed +resource "alicloud_instance" "ecs_instance" { + instance_name = "ecs-${local.common_name}" + image_id = data.alicloud_images.default.images[0].id + instance_type = var.instance_type + system_disk_category = "cloud_essd" + security_groups = [alicloud_security_group.security_group.id] + vswitch_id = alicloud_vswitch.vswitch.id + password = var.ecs_instance_password + internet_max_bandwidth_out = 5 +} + +resource "alicloud_ecs_command" "run_command" { + name = "command-genlog-loongcollector-${local.common_name}" + command_content = base64encode(<> ~/.bash_profile +export ROS_DEPLOY=true +export ALIBABA_CLOUD_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id} +export ALIBABA_CLOUD_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret} +EOT + +source ~/.bash_profile +curl -fsSL https://help-static-aliyun-doc.aliyuncs.com/tech-solution/install-log-monitoring-alarming-0.1.sh|bash +wget http://aliyun-observability-release-${var.region}.oss-${var.region}.aliyuncs.com/loongcollector/linux64/latest/loongcollector.sh -O loongcollector.sh +chmod +x loongcollector.sh +./loongcollector.sh install ${var.region}-internet +EOF + ) + working_dir = "/root" + type = "RunShellScript" + timeout = 3600 +} + +resource "alicloud_ecs_invocation" "invoke_script" { + instance_id = [alicloud_instance.ecs_instance.id] + command_id = alicloud_ecs_command.run_command.id + timeouts { + create = "15m" + } + depends_on = [alicloud_instance.ecs_instance] +} + +resource "alicloud_log_project" "sls_project" { + project_name = "sls-project-${local.common_name}" +} + +resource "alicloud_log_store" "sls_log_store" { + logstore_name = "sls-logstore-${local.common_name}" + project_name = alicloud_log_project.sls_project.project_name + depends_on = [alicloud_log_project.sls_project] +} + +resource "alicloud_log_machine_group" "this" { + identify_list = [alicloud_instance.ecs_instance.primary_ip_address] + name = "lmg-${local.common_name}" + project = alicloud_log_project.sls_project.project_name + identify_type = "ip" +} + +resource "alicloud_logtail_config" "this" { + project = alicloud_log_project.sls_project.project_name + input_detail = <> ~/.bash_profile +export ROS_DEPLOY=true +export ALIBABA_CLOUD_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id} +export ALIBABA_CLOUD_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret} +EOT + +source ~/.bash_profile + +# 安装Docker +yum install -y yum-utils device-mapper-persistent-data lvm2 +yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo +yum -y install docker-ce +docker --version +systemctl start docker +systemctl enable docker + +# 创建项目路径和用于存放数据的目录 +mkdir sls-kibana +cd sls-kibana +mkdir data +chmod 777 data + +# 在项目路径下创建.env文件 +cat << EOJ >> .env +ES_PASSWORD=${var.ecs_instance_password} +SLS_ENDPOINT=${var.region}.log.aliyuncs.com +SLS_PROJECT=${alicloud_log_project.sls_project.project_name} +# 需要提前创建RAM用户，且需要为RAM用户授予Logstore的查询权限 +# ECS RAM角色，请参见：https://help.aliyun.com/zh/ecs/user-guide/attach-an-instance-ram-role-to-an-ecs-instance +# ECS RAM角色授权，请参见：https://help.aliyun.com/zh/sls/compatibility-between-log-service-and-elasticsearch#de61167fc0lqi +SLS_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id} +SLS_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret} +EOJ + +# 在项目路径下创建docker-compose.yaml文件 +cat << EOK >> docker-compose.yaml +services: + es: + image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/elasticsearch:7.17.26 + environment: + - "discovery.type=single-node" + - "ES_JAVA_OPTS=-Xms2G -Xmx2G" + - ELASTIC_USERNAME=elastic + - ELASTIC_PASSWORD=${var.ecs_instance_password} + - xpack.security.enabled=true + volumes: + - ./data:/usr/share/elasticsearch/data + kproxy: + image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/kproxy:2.1.4 + depends_on: + - es + environment: + - ES_ENDPOINT=es:9200 + - SLS_ENDPOINT=${var.region}.log.aliyuncs.com + - SLS_PROJECT=${alicloud_log_project.sls_project.project_name} + - SLS_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id} + - SLS_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret} + kibana: + image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/kibana:7.17.26 + depends_on: + - kproxy + environment: + - ELASTICSEARCH_HOSTS=http://kproxy:9201 + - ELASTICSEARCH_USERNAME=elastic + - ELASTICSEARCH_PASSWORD=${var.ecs_instance_password} + - XPACK_MONITORING_UI_CONTAINER_ELASTICSEARCH_ENABLED=true + ports: + - "5601:5601" + # 这个服务组件是可选的，作用是自动创建kibana index pattern + index-patterner: + image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/kproxy:2.1.4 + command: /usr/bin/python3 -u /workspace/create_index_pattern.py + depends_on: + - kibana + environment: + - KPROXY_ENDPOINT=http://kproxy:9201 + - KIBANA_ENDPOINT=http://kibana:5601 + - KIBANA_USER=elastic + - KIBANA_PASSWORD=${var.ecs_instance_password} + - SLS_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id} + - SLS_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret} +EOK + +# 启动Kibana +docker compose up -d +docker compose ps +EOF + ) + working_dir = "/root" + type = "RunShellScript" + timeout = 3600 +} + +resource "alicloud_ecs_invocation" "invoke_script_kibana" { + instance_id = [alicloud_instance.ecs_instance_kibana.id] + command_id = alicloud_ecs_command.run_command_kibana.id + timeouts { + create = "15m" + } + depends_on = [alicloud_instance.ecs_instance_kibana] +} diff --git a/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/outputs.tf b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/outputs.tf new file mode 100644 index 0000000000..6cf5def191 --- /dev/null +++ b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/outputs.tf @@ -0,0 +1,14 @@ +output "ecs_login_address" { + description = "生成日志的ECS实例的登录地址。通过此地址登录ECS后，在本地查看生成日志文件的命令为：tail -f /tmp/sls-monitor-test.log" + value = format("https://ecs-workbench.aliyun.com/?from=ecs&instanceType=ecs®ionId=%s&instanceId=%s&resourceGroupId=", var.region, alicloud_instance.ecs_instance.id) +} + +output "sls_logsearch_url" { + description = "SLS日志查询入口" + value = format("https://sls.console.aliyun.com/lognext/project/%s/logsearch/%s?slsRegion=%s", alicloud_log_project.sls_project.project_name, alicloud_log_store.sls_log_store.logstore_name, var.region) +} + +output "kibana_management_url" { + description = "Kibana管理界面入口，登录用户名为elastic，登录密码为您在配置时传入的密码" + value = format("http://%s:5601", alicloud_instance.ecs_instance_kibana.public_ip) +} diff --git a/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/variables.tf b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/variables.tf new file mode 100644 index 0000000000..0fc7159757 --- /dev/null +++ b/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform/variables.tf @@ -0,0 +1,22 @@ +variable "region" { + type = string + default = "cn-hangzhou" +} + +variable "instance_type" { + type = string + default = "ecs.e-c1m2.large" + description = "实例类型" +} + +variable "instance_type_xlarge" { + type = string + default = "ecs.e-c1m2.xlarge" + description = "高性能实例类型" +} + +variable "ecs_instance_password" { + type = string + sensitive = true + description = "服务器登录密码,长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）" +}