Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
## Introduction

<!-- DOCS_DESCRIPTION_CN -->
本示例用于实现解决方案[开源自建ELK上云指南:基于阿里云日志服务(SLS)构建低成本可扩展日志平台](https://www.aliyun.com/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform), 涉及到专有网络(VPC)、交换机(VSwitch)、云服务器(ECS)、RAM 用户等资源的创建。
<!-- DOCS_DESCRIPTION_CN -->

<!-- DOCS_DESCRIPTION_EN -->
This example demonstrates the implementation of the solution [Build Large Scale Low Cost Realtime Log Management Platform](https://www.aliyun.com/solution/tech-solution/build-large-scale-low-cost-real-time-log-management-platform). It involves the creation, and deployment of resources such as Virtual Private Cloud (VPC), VSwitch, Elastic Compute Service (ECS), and RAM users.
<!-- DOCS_DESCRIPTION_EN -->

<!-- BEGIN_TF_DOCS -->
## Providers

| Name | Version |
|------|---------|
| <a name="provider_alicloud"></a> [alicloud](#provider\_alicloud) | n/a |
| <a name="provider_random"></a> [random](#provider\_random) | n/a |

## Modules

No modules.

## Resources

| Name | Type |
|------|------|
| [alicloud_ecs_command.run_command](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_command) | resource |
| [alicloud_ecs_command.run_command_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_command) | resource |
| [alicloud_ecs_invocation.invoke_script](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_invocation) | resource |
| [alicloud_ecs_invocation.invoke_script_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ecs_invocation) | resource |
| [alicloud_instance.ecs_instance](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/instance) | resource |
| [alicloud_instance.ecs_instance_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/instance) | resource |
| [alicloud_log_machine_group.this](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_machine_group) | resource |
| [alicloud_log_project.sls_project](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_project) | resource |
| [alicloud_log_store.sls_log_store](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_store) | resource |
| [alicloud_log_store_index.sls_index](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/log_store_index) | resource |
| [alicloud_logtail_attachment.this](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/logtail_attachment) | resource |
| [alicloud_logtail_config.this](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/logtail_config) | resource |
| [alicloud_ram_access_key.ramak](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ram_access_key) | resource |
| [alicloud_ram_user.ram_user](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ram_user) | resource |
| [alicloud_ram_user_policy_attachment.attach_policy_to_user](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/ram_user_policy_attachment) | resource |
| [alicloud_security_group.security_group](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/security_group) | resource |
| [alicloud_security_group.security_group_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/security_group) | resource |
| [alicloud_security_group_rule.allow_kibana](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/security_group_rule) | resource |
| [alicloud_vpc.vpc](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/vpc) | resource |
| [alicloud_vswitch.vswitch](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/resources/vswitch) | resource |
| [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
| [alicloud_images.default](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/data-sources/images) | data source |
| [alicloud_zones.default](https://registry.terraform.io/providers/hashicorp/alicloud/latest/docs/data-sources/zones) | data source |

## Inputs

| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_ecs_instance_password"></a> [ecs\_instance\_password](#input\_ecs\_instance\_password) | 服务器登录密码,长度8-30,必须包含三项(大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号)` | `string` | n/a | yes |
| <a name="input_instance_type"></a> [instance\_type](#input\_instance\_type) | 实例类型 | `string` | `"ecs.e-c1m2.large"` | no |
| <a name="input_instance_type_xlarge"></a> [instance\_type\_xlarge](#input\_instance\_type\_xlarge) | 高性能实例类型 | `string` | `"ecs.e-c1m2.xlarge"` | no |
| <a name="input_region"></a> [region](#input\_region) | n/a | `string` | `"cn-hangzhou"` | no |
<!-- END_TF_DOCS -->
Original file line number Diff line number Diff line change
@@ -0,0 +1,292 @@
provider "alicloud" {
region = var.region
}

data "alicloud_zones" "default" {
available_disk_category = "cloud_essd"
available_resource_creation = "VSwitch"
available_instance_type = var.instance_type
}

resource "random_string" "suffix" {
length = 8
lower = true
upper = false
numeric = false
special = false
}

locals {
common_name = random_string.suffix.id
}

resource "alicloud_vpc" "vpc" {
cidr_block = "192.168.0.0/16"
vpc_name = "vpc-${local.common_name}"
}

resource "alicloud_vswitch" "vswitch" {
vpc_id = alicloud_vpc.vpc.id
cidr_block = "192.168.0.0/24"
zone_id = data.alicloud_zones.default.zones.0.id
vswitch_name = "vswitch-${local.common_name}"
}

resource "alicloud_security_group" "security_group" {
vpc_id = alicloud_vpc.vpc.id
security_group_name = "sg-${local.common_name}"
}

data "alicloud_images" "default" {
name_regex = "^aliyun_3_x64_20G_alibase_.*"
most_recent = true
owners = "system"
}

resource "alicloud_ram_user" "ram_user" {
name = "create_by_solution-${local.common_name}"
}

resource "alicloud_ram_access_key" "ramak" {
user_name = alicloud_ram_user.ram_user.name
depends_on = [
alicloud_ram_user.ram_user
]
}

resource "alicloud_ram_user_policy_attachment" "attach_policy_to_user" {
user_name = alicloud_ram_user.ram_user.name
policy_type = "System"
policy_name = "AliyunLogFullAccess"
depends_on = [
alicloud_ram_access_key.ramak
]
}

# the ECS instance which generate the log, and where LoongCollector is installed
resource "alicloud_instance" "ecs_instance" {
instance_name = "ecs-${local.common_name}"
image_id = data.alicloud_images.default.images[0].id
instance_type = var.instance_type
system_disk_category = "cloud_essd"
security_groups = [alicloud_security_group.security_group.id]
vswitch_id = alicloud_vswitch.vswitch.id
password = var.ecs_instance_password
internet_max_bandwidth_out = 5
}

resource "alicloud_ecs_command" "run_command" {
name = "command-genlog-loongcollector-${local.common_name}"
command_content = base64encode(<<EOF
cat << EOT >> ~/.bash_profile
export ROS_DEPLOY=true
export ALIBABA_CLOUD_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id}
export ALIBABA_CLOUD_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret}
EOT

source ~/.bash_profile
curl -fsSL https://help-static-aliyun-doc.aliyuncs.com/tech-solution/install-log-monitoring-alarming-0.1.sh|bash
wget http://aliyun-observability-release-${var.region}.oss-${var.region}.aliyuncs.com/loongcollector/linux64/latest/loongcollector.sh -O loongcollector.sh
chmod +x loongcollector.sh
./loongcollector.sh install ${var.region}-internet
EOF
)
working_dir = "/root"
type = "RunShellScript"
timeout = 3600
}

resource "alicloud_ecs_invocation" "invoke_script" {
instance_id = [alicloud_instance.ecs_instance.id]
command_id = alicloud_ecs_command.run_command.id
timeouts {
create = "15m"
}
depends_on = [alicloud_instance.ecs_instance]
}

resource "alicloud_log_project" "sls_project" {
project_name = "sls-project-${local.common_name}"
}

resource "alicloud_log_store" "sls_log_store" {
logstore_name = "sls-logstore-${local.common_name}"
project_name = alicloud_log_project.sls_project.project_name
depends_on = [alicloud_log_project.sls_project]
}

resource "alicloud_log_machine_group" "this" {
identify_list = [alicloud_instance.ecs_instance.primary_ip_address]
name = "lmg-${local.common_name}"
project = alicloud_log_project.sls_project.project_name
identify_type = "ip"
}

resource "alicloud_logtail_config" "this" {
project = alicloud_log_project.sls_project.project_name
input_detail = <<EOF
{
"discardUnmatch": false,
"enableRawLog": true,
"fileEncoding": "utf8",
"filePattern": "sls-monitor-test.log",
"logPath": "/tmp",
"logType": "common_reg_log",
"maxDepth": 10,
"topicFormat": "none"
}
EOF
input_type = "file"
logstore = alicloud_log_store.sls_log_store.logstore_name
name = "lc-${local.common_name}"
output_type = "LogService"
}

resource "alicloud_logtail_attachment" "this" {
project = alicloud_log_project.sls_project.project_name
logtail_config_name = alicloud_logtail_config.this.name
machine_group_name = alicloud_log_machine_group.this.name
}

resource "alicloud_log_store_index" "sls_index" {
project = alicloud_log_project.sls_project.project_name
logstore = alicloud_log_store.sls_log_store.logstore_name
full_text {}
field_search {
name = "content"
type = "text"
}
}

resource "alicloud_security_group" "security_group_kibana" {
vpc_id = alicloud_vpc.vpc.id
security_group_name = "sg-kibana-${local.common_name}"
}

resource "alicloud_security_group_rule" "allow_kibana" {
type = "ingress"
ip_protocol = "tcp"
nic_type = "intranet"
policy = "accept"
port_range = "5601/5601"
priority = 1
security_group_id = alicloud_security_group.security_group_kibana.id
cidr_ip = "0.0.0.0/0"
}

# the ECS instance where Kibana is deployed
resource "alicloud_instance" "ecs_instance_kibana" {
instance_name = "ecs-kibana-${local.common_name}"
image_id = data.alicloud_images.default.images[0].id
instance_type = var.instance_type_xlarge
system_disk_category = "cloud_essd"
security_groups = [alicloud_security_group.security_group_kibana.id]
vswitch_id = alicloud_vswitch.vswitch.id
password = var.ecs_instance_password
internet_max_bandwidth_out = 10
}

resource "alicloud_ecs_command" "run_command_kibana" {
name = "command-kibana-${local.common_name}"
command_content = base64encode(<<EOF
cat << EOT >> ~/.bash_profile
export ROS_DEPLOY=true
export ALIBABA_CLOUD_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id}
export ALIBABA_CLOUD_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret}
EOT

source ~/.bash_profile

# 安装Docker
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum -y install docker-ce
docker --version
systemctl start docker
systemctl enable docker

# 创建项目路径和用于存放数据的目录
mkdir sls-kibana
cd sls-kibana
mkdir data
chmod 777 data

# 在项目路径下创建.env文件
cat << EOJ >> .env
ES_PASSWORD=${var.ecs_instance_password}
SLS_ENDPOINT=${var.region}.log.aliyuncs.com
SLS_PROJECT=${alicloud_log_project.sls_project.project_name}
# 需要提前创建RAM用户,且需要为RAM用户授予Logstore的查询权限
# ECS RAM角色,请参见:https://help.aliyun.com/zh/ecs/user-guide/attach-an-instance-ram-role-to-an-ecs-instance
# ECS RAM角色授权,请参见:https://help.aliyun.com/zh/sls/compatibility-between-log-service-and-elasticsearch#de61167fc0lqi
SLS_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id}
SLS_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret}
EOJ

# 在项目路径下创建docker-compose.yaml文件
cat << EOK >> docker-compose.yaml
services:
es:
image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/elasticsearch:7.17.26
environment:
- "discovery.type=single-node"
- "ES_JAVA_OPTS=-Xms2G -Xmx2G"
- ELASTIC_USERNAME=elastic
- ELASTIC_PASSWORD=${var.ecs_instance_password}
- xpack.security.enabled=true
volumes:
- ./data:/usr/share/elasticsearch/data
kproxy:
image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/kproxy:2.1.4
depends_on:
- es
environment:
- ES_ENDPOINT=es:9200
- SLS_ENDPOINT=${var.region}.log.aliyuncs.com
- SLS_PROJECT=${alicloud_log_project.sls_project.project_name}
- SLS_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id}
- SLS_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret}
kibana:
image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/kibana:7.17.26
depends_on:
- kproxy
environment:
- ELASTICSEARCH_HOSTS=http://kproxy:9201
- ELASTICSEARCH_USERNAME=elastic
- ELASTICSEARCH_PASSWORD=${var.ecs_instance_password}
- XPACK_MONITORING_UI_CONTAINER_ELASTICSEARCH_ENABLED=true
ports:
- "5601:5601"
# 这个服务组件是可选的,作用是自动创建kibana index pattern
index-patterner:
image: sls-registry.cn-hangzhou.cr.aliyuncs.com/kproxy/kproxy:2.1.4
command: /usr/bin/python3 -u /workspace/create_index_pattern.py
depends_on:
- kibana
environment:
- KPROXY_ENDPOINT=http://kproxy:9201
- KIBANA_ENDPOINT=http://kibana:5601
- KIBANA_USER=elastic
- KIBANA_PASSWORD=${var.ecs_instance_password}
- SLS_ACCESS_KEY_ID=${alicloud_ram_access_key.ramak.id}
- SLS_ACCESS_KEY_SECRET=${alicloud_ram_access_key.ramak.secret}
EOK

# 启动Kibana
docker compose up -d
docker compose ps
EOF
)
working_dir = "/root"
type = "RunShellScript"
timeout = 3600
}

resource "alicloud_ecs_invocation" "invoke_script_kibana" {
instance_id = [alicloud_instance.ecs_instance_kibana.id]
command_id = alicloud_ecs_command.run_command_kibana.id
timeouts {
create = "15m"
}
depends_on = [alicloud_instance.ecs_instance_kibana]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
output "ecs_login_address" {
description = "生成日志的ECS实例的登录地址。通过此地址登录ECS后,在本地查看生成日志文件的命令为:tail -f /tmp/sls-monitor-test.log"
value = format("https://ecs-workbench.aliyun.com/?from=ecs&instanceType=ecs&regionId=%s&instanceId=%s&resourceGroupId=", var.region, alicloud_instance.ecs_instance.id)
}

output "sls_logsearch_url" {
description = "SLS日志查询入口"
value = format("https://sls.console.aliyun.com/lognext/project/%s/logsearch/%s?slsRegion=%s", alicloud_log_project.sls_project.project_name, alicloud_log_store.sls_log_store.logstore_name, var.region)
}

output "kibana_management_url" {
description = "Kibana管理界面入口,登录用户名为elastic,登录密码为您在配置时传入的密码"
value = format("http://%s:5601", alicloud_instance.ecs_instance_kibana.public_ip)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
variable "region" {
type = string
default = "cn-hangzhou"
}

variable "instance_type" {
type = string
default = "ecs.e-c1m2.large"
description = "实例类型"
}

variable "instance_type_xlarge" {
type = string
default = "ecs.e-c1m2.xlarge"
description = "高性能实例类型"
}

variable "ecs_instance_password" {
type = string
sensitive = true
description = "服务器登录密码,长度8-30,必须包含三项(大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号)"
}
Loading