root@tr-desktop:/home/tr/elk# tree
.
├── docker-compose.yml
├── logstash
│ ├── config
│ │ └── logstash.yml
│ ├── jar
│ │ └── mysql-connector-j-9.2.0.jar
│ └── pipeline
│ ├── logstash.conf
│ └── mysql_to_es.conf
└── mysql-connector-j-9.2.0.tar.gz
4 directories, 6 files
root@tr-desktop:/home/tr/elk# cat docker-compose.yml
version: '3.7'
services:
# Elasticsearch 服务
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.6.0
environment:
- discovery.type=single-node
- xpack.security.enabled=false
# - ELASTIC_PASSWORD=changeme # 设置密码（可以更改）
ports:
- "9200:9200"
networks:
- elk
volumes:
- esdata:/usr/share/elasticsearch/data
ulimits:
memlock:
soft: -1
hard: -1
# Logstash 服务
logstash:
image: docker.elastic.co/logstash/logstash:8.6.0
environment:
- XPACK_SECURITY_ENABLED=false # 禁用安全认证（根据需要启用）
- LS_JAVA_OPTS=-Xmx2g -Xms2g
ports:
- "5044:5044"
networks:
- elk
volumes:
- ./logstash/config:/usr/share/logstash/config
- ./logstash/pipeline:/usr/share/logstash/pipeline
#- ./logstash/jar:/usr/share/logstash/logstash-core/lib/jars # 挂载 JDBC 驱动
# Kibana 服务
kibana:
image: docker.elastic.co/kibana/kibana:8.6.0
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
ports:
- "5601:5601"
networks:
- elk
depends_on:
- elasticsearch
networks:
elk:
driver: bridge
volumes:
esdata:
driver: local
root@tr-desktop:/home/tr/elk# cat logstash/config/logstash.yml
http.host: "0.0.0.0"
path.config: "/usr/share/logstash/pipeline"
path.data: "/usr/share/logstash/data"
xpack.monitoring.enabled: false
log.level: info
root@tr-desktop:/home/tr/elk# cat logstash/pipeline/mysql_to_es.conf
input {
jdbc {
jdbc_connection_string => "jdbc:mysql://10.1.8.206:3306/troa_new"
jdbc_user => "root"
jdbc_password => "RzaikJAM5sSxmZhT"
jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
jdbc_driver_library => "/usr/share/logstash/logstash-core/lib/jars/mysql-connector-j-9.2.0.jar" # MySQL JDBC驱动的路径
statement => "SELECT * FROM tbl_import_express" # MySQL查询语句
schedule => "* * * * *" # 每分钟查询一次，具体根据需求设置
}
}
filter {
# 可以添加一些过滤操作，如果需要对数据进行修改或转换
# 比如你可以使用 date 或 mutate 插件来处理数据
}
output {
elasticsearch {
hosts => ["http://10.1.8.206:9200"] # Elasticsearch的地址
index => "my_index" # Elasticsearch索引名
#user => "your_es_username" # 如果Elasticsearch需要认证
#password => "your_es_password" # Elasticsearch认证密码
document_id => "%{id}" # 设置id字段为Elasticsearch的文档ID（根据需求选择）
}
stdout { codec => rubydebug } # 用于调试时打印日志输出
}
root@tr-desktop:/home/tr/elk# cat logstash/pipeline/
logstash.conf mysql_to_es.conf
root@tr-desktop:/home/tr/elk# cat logstash/pipeline/
logstash.conf mysql_to_es.conf
root@tr-desktop:/home/tr/elk# cat logstash/pipeline/logstash.conf
input {
file {
path => "/usr/share/logstash/input/*.log"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
index => "logstash-%{+YYYY.MM.dd}"
}
}
root@tr-desktop:/home/tr/elk#
