Elastic Stack之FileBeat使用实战
作者：尹正杰
版权声明：原创作品，谢绝转载！否则将追究法律责任。
本篇博客数据流走向：FileBeat ===》logstash ===> elasticsearch。
一.安装FileBeat
1>.选择FileBeat版本（连接地址：https://www.elastic.co/downloads/beats）
2>.选择之前的发行版本
3>.选择FileBeat版本
4>.下载FileBeat（连接地址：https://www.elastic.co/downloads/past-releases/filebeat-5-6-12）
[root@node105 ~]# wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.12-x86_64.rpm --2019-03-11 22:21:04-- https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.6.12-x86_64.rpm Resolving artifacts.elastic.co (artifacts.elastic.co)... 151.101.230.222, 2a04:4e42:1a::734 Connecting to artifacts.elastic.co (artifacts.elastic.co)|151.101.230.222|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 9224611 (8.8M) [application/octet-stream] Saving to: ‘filebeat-5.6.12-x86_64.rpm’ 100%[===========================================================================================================================================================>] 9,224,611 63.9KB/s in 2m 2s 2019-03-11 22:23:09 (74.0 KB/s) - ‘filebeat-5.6.12-x86_64.rpm’ saved [9224611/9224611] [root@node105 ~]#
二.使用FileBeat的案例实战
1>.安装FileBeat
[root@node105 ~]# [root@node105 ~]# ll total 139728 -rw-r--r--. 1 root root 9224611 Sep 19 03:10 filebeat-5.6.12-x86_64.rpm -rw-r--r--. 1 root root 28513410 Mar 4 23:29 GeoLite2-City.tar.gz -rw-r--r--. 1 root root 105333923 Sep 19 03:12 logstash-5.6.12.rpm [root@node105 ~]# [root@node105 ~]# [root@node105 ~]# rpm -ivh filebeat-5.6.12-x86_64.rpm warning: filebeat-5.6.12-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY Preparing... ################################# [100%] Updating / installing... 1:filebeat-5.6.12-1 ################################# [100%] [root@node105 ~]# [root@node105 ~]#
[root@node105 ~]# [root@node105 ~]# rpm -ql filebeat /etc/filebeat/filebeat.full.yml /etc/filebeat/filebeat.template-es2x.json /etc/filebeat/filebeat.template-es6x.json /etc/filebeat/filebeat.template.json /etc/filebeat/filebeat.yml /etc/init.d/filebeat /lib/systemd/system/filebeat.service /usr/bin/filebeat.sh /usr/share/filebeat/.build_hash.txt /usr/share/filebeat/NOTICE /usr/share/filebeat/README.md /usr/share/filebeat/bin/filebeat /usr/share/filebeat/bin/filebeat-god /usr/share/filebeat/module/apache2/access/config/access.yml /usr/share/filebeat/module/apache2/access/ingest/default.json /usr/share/filebeat/module/apache2/access/manifest.yml /usr/share/filebeat/module/apache2/error/config/error.yml /usr/share/filebeat/module/apache2/error/ingest/pipeline.json /usr/share/filebeat/module/apache2/error/manifest.yml /usr/share/filebeat/module/auditd/log/config/log.yml /usr/share/filebeat/module/auditd/log/ingest/pipeline.json /usr/share/filebeat/module/auditd/log/manifest.yml /usr/share/filebeat/module/mysql/error/config/error.yml /usr/share/filebeat/module/mysql/error/ingest/pipeline.json /usr/share/filebeat/module/mysql/error/manifest.yml /usr/share/filebeat/module/mysql/slowlog/config/slowlog.yml /usr/share/filebeat/module/mysql/slowlog/ingest/pipeline.json /usr/share/filebeat/module/mysql/slowlog/manifest.yml /usr/share/filebeat/module/nginx/access/config/nginx-access.yml /usr/share/filebeat/module/nginx/access/ingest/default.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_low_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_remote_ip_url_count.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_response_code.json /usr/share/filebeat/module/nginx/access/machine_learning/datafeed_visitor_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/low_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json /usr/share/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json /usr/share/filebeat/module/nginx/access/machine_learning/response_code.json /usr/share/filebeat/module/nginx/access/machine_learning/visitor_rate.json /usr/share/filebeat/module/nginx/access/manifest.yml /usr/share/filebeat/module/nginx/error/config/nginx-error.yml /usr/share/filebeat/module/nginx/error/ingest/pipeline.json /usr/share/filebeat/module/nginx/error/manifest.yml /usr/share/filebeat/module/system/auth/config/auth.yml /usr/share/filebeat/module/system/auth/ingest/pipeline.json /usr/share/filebeat/module/system/auth/manifest.yml /usr/share/filebeat/module/system/syslog/config/syslog.yml /usr/share/filebeat/module/system/syslog/ingest/pipeline.json /usr/share/filebeat/module/system/syslog/manifest.yml /usr/share/filebeat/scripts/import_dashboards /usr/share/filebeat/scripts/migrate_beat_config_1_x_to_5_0.py [root@node105 ~]#
2>.修改FileBeat的配置文件
[root@node105 ~]# [root@node105 ~]# [root@node105 ~]# cp /etc/filebeat/filebeat.yml{,.bak} [root@node105 ~]# [root@node105 ~]# ll /etc/filebeat/ total 124 -rw-r--r--. 1 root root 38827 Sep 11 04:06 filebeat.full.yml -rw-r--r--. 1 root root 25088 Sep 11 04:06 filebeat.template-es2x.json -rw-r--r--. 1 root root 20028 Sep 11 04:06 filebeat.template-es6x.json -rw-r--r--. 1 root root 20028 Sep 11 04:06 filebeat.template.json -rw-------. 1 root root 4196 Sep 11 04:06 filebeat.yml -rw-------. 1 root root 4196 Mar 11 22:26 filebeat.yml.bak [root@node105 ~]# [root@node105 ~]#
[root@node105 ~]# [root@node105 ~]# cat /etc/filebeat/filebeat.yml | egrep -v "^#|^$| #" filebeat.prospectors: - input_type: log paths: - /var/log/httpd/access_log output.logstash: hosts: ["node105.yinzhengjie.org.cn:5044"] [root@node105 ~]# [root@node105 ~]#
3>.修改Logstash的配置文件，配置输入插件为beats并启动（参考链接：https://www.elastic.co/guide/en/logstash/5.6/plugins-inputs-beats.html）
[root@node105 conf.d]# [root@node105 conf.d]# cat beats-filter-elasticsearch.conf input { beats { port =>5044 } } filter { grok { match => { "message" => "%{HTTPD_COMBINEDLOG}" } remove_field => "message" } date { match => ["timestamp","dd/MMM/YYYY:H:m:s Z"] remove_field => "timestamp" } geoip { source => "clientip" target => "geoip" database => "/etc/logstash/maxmind/GeoLite2-City.mmdb" } mutate { rename => { "agent" => "user_agent" } } } output { elasticsearch { hosts => ["http://node101.yinzhengjie.org.cn:9200/","http://node102.yinzhengjie.org.cn:9200/","http://node103.yinzhengjie.org.cn:9200/"] index => "logstash-%{+YYYY.MM.dd}" document_type => "httpd_access_logs" } } [root@node105 conf.d]# [root@node105 conf.d]# [root@node105 conf.d]# logstash -f beats-filter-elasticsearch.conf -t WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console Configuration OK [root@node105 conf.d]#
京公网安备 11010802041100号