logstash收集日志并写入Redis再到es集群

redis做数据缓存

图形架构：

环境准备

172.31.2.101 es1 + kibana
172.31.2.102 es2
172.31.2.103 es3
172.31.2.104 logstash1
172.31.2.105 logstash2
172.31.2.106 Redis
172.31.2.107 web1


安装redis

[root@es-redis ~]# apt install redis -y


改redis 配置

[root@es-redis ~]# vim /etc/redis/redis.conf
bind 0.0.0.0
requirepass 123456
save ""
#save 900 1
#save 300 10
#save 60 10000


重启

[root@es-redis ~]# systemctl restart redis


检查端口

[root@es-redis ~]# ss -tnl
6379


在web服务器Nginx-logstash配置改如下

建议把host写上

[root@es-redis ~]# vim /etc/logstash/conf.d/nginx-log-es.conf
input{
file{
path => "/var/log/nginx/access.log"
start_position => "beginning"
stat_interval => 3
type => "nginx-accesslog"
codec => "json"
}
}
output{
if [type] == "nginx-accesslog" {
redis {
data_type => "list"
host => "172.31.2.106"
key => "nginx-accesslog"
port => "6379"
db => "1"
password => "123456"
}}
}


重启

[root@es-redis ~]# systemctl restart logstash


访问nginx让其产生数据

在redis服务器测试

[root@es-redis ~]# redis-cli -h 172.31.2.106
172.31.2.106:6379> AUTH 123456
OK
172.31.2.106:6379> SELECT 1
OK
172.31.2.106:6379[1]> keys *
1) "nginx-accesslog"
172.31.2.106:6379[1]> LPOP nginx-accesslog


logstash服务器写到es 的配置

[root@es-web1 ~]# vim nginx-log-es.conf
input {
redis {
data_type => "list"
key => "nginx-accesslog"
host => "172.31.2.106"
port => "6379"
db => "1"
password => "123456"
codec => "json"
}
}
output {
if [type] == "nginx-accesslog" {
elasticsearch{
hosts => ["172.31.2.101:9200"]
index => "n826-long-nginx-accesslog-%{+YYYY.MM.dd}"
}}
}


停止

[root@es-redis ~]# systemctl stop logstash.service


运行

[root@es-redis ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-log-es.conf


添加到kibana

略

创建视图

把Nginx错误日志也配置

[root@es-web1 ~]# cat /etc/logstash/conf.d/nginx-log-es.conf
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
stat_interval => 3
type => "nginx-accesslog"
codec => "json"
}
file {
path => "/apps/nginx/logs/error.log"
start_position => "beginning"
stat_interval => 3
type => "nginx-errorlog"
#codec => "json"
}
}
output {
if [type] == "nginx-accesslog" {
redis {
data_type => "list"
host => "172.31.2.106"
key => "nginx-accesslog"
port => "6379"
db => "1"
password => "123456"
}}
if [type] == "nginx-errorlog" {
redis {
data_type => "list"
host => "172.31.2.106"
key => "nginx-errorlog"
port => "6379"
db => "1"
password => "123456"
}}
}


重启

[root@es-redis ~]# systemctl restart logstash


制作错误日志信息

[root@es-web1 ~]# echo "error 654321 web" >> /apps/nginx/logs/error.log
[root@es-web1 ~]# echo "error 123456 web" >> /apps/nginx/logs/error.log


在把logstash写入es集群

[root@logstash1 ~]# cat /etc/logstash/conf.d/nginx-log-es.conf
input {
redis {
data_type => "list"
key => "nginx-accesslog"
host => "172.31.2.106"
port => "6379"
db => "1"
password => "123456"
codec => "json"
}
redis {
data_type => "list"
key => "nginx-errorlog"
host => "172.31.2.106"
port => "6379"
db => "1"
password => "123456"
}
}
output {
if [type] == "nginx-accesslog" {
elasticsearch {
hosts => ["172.31.2.101:9200"]
index => "n826-long-nginx-accesslog-%{+YYYY.MM.dd}"
}}
if [type] == "nginx-errorlog" {
elasticsearch {
hosts => ["172.31.2.101:9200"]
index => "n826-long-nginx-errorlog-%{+YYYY.MM.dd}"
}}
}


重启

[root@es-redis ~]# systemctl restart logstash


当logstash去redis取数据，redis就会没有，如果数据多的话取一次就会少一次

原文链接：https://www.cnblogs.com/xuanlv-0413/p/15374797.html