A Simple & Powerful Log Aggregation Pipeline with Logstash and Filebeat

Requirements

Deployment Process

Deployment diagram
input {
beats {
port => "5044"
}
}
filter {
mutate {
add_field => {
"agent_host" => "%{[host][name]}"
"agent_src_path" => "%{[log][file][path]}"
}
}
mutate {
split => {
"agent_src_path" => "/"
}
}
ruby {
code => "
last_idx = event.get('agent_src_path').length-1;
event.set('agent_src_path_last_index', last_idx);
event.set('agent_log_file_name', event.get('agent_src_path')[last_idx]);
"
}
mutate {
remove_field => [ "[agent_src_path][%{agent_src_path_last_index}]", "[agent_src_path][0]" ]
}
mutate {
join => { "agent_src_path" => "-" }
remove_field => ["agent_src_path_last_index"]
}
}
output {
stdout {
#codec => line
}
file {
path => "/home/centralized-log/%{+YYYY-MM-dd}/%{agent_host}/%{agent_src_path}/%{agent_log_file_name}"
codec => line { format => "%{message}"}
}
}
#!/bin/bashJAVA_HOME=/home/jdk
export JAVA_HOME
cd /home/logstash-7.6.0
./bin/logstash -f config/app.conf --config.reload.automatic >/dev/null 2>&1 &
#!/bin/bashpid=`pgrep -f logstash`
kill -9 $pid
filebeat.inputs:
- type: log
paths:
# general system logs
- /var/log/messages*
- /var/log/audit/audit.log*
# your application log path here
# ...
# ...
logging.metrics.enabled: falseoutput.logstash:
hosts: ["192.168.1.1:5044"]
#!/bin/bashcd /home/filebeat-7.6.0-linux-x86_64
./filebeat -c app.yml -d "publish" >/dev/null 2>&1 &
#!/bin/bashpid=`pgrep -f filebeat`
kill -9 $pid
# tree /home/centralized-log/ -d
/home/centralized-log/
├── 2020-03-17
│ ├── app1
│ │ ├── home-RestApi-cython-dist-logs
│ │ └── var-log
│ ├── app2
│ │ ├── home-RestApi-cython-dist-logs
│ │ └── var-log
├── 2020-03-18
│ ├── app1
│ │ ├── home-RestApi-cython-dist-logs
│ │ └── var-log
│ ├── app2
│ │ ├── home-RestApi-cython-dist-logs
│ │ └── var-log

Last Thoughts

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store