20.1 下载
<u>https://www.elastic.co/cn/downloads/logstash</u>
logstash-8.0.0-linux-x86_64.tar.gz
tar -zxvf logstash-8.0.0-linux-x86_64.tar.gz
mv logstash-8.0.0-linux-x86_64 logstash
mkdir -p /home/software/logstash/data
mkdir -p /home/software/logstash/logs
20.2 config
vim config/input-output.conf
input{
kafka{
bootstrap_servers => "172.16.10.4:9092,172.16.10.6:9092,172.16.10.7:9092"
topics => ["zysco_realtime"]
group_id => "logstash1"
type => "realtime"
}
}
filter{
grok{
match => ["message",""(?<data>([\S+]*))""]
}
}
output{
kafka{
bootstrap_servers => "localhost:9092"
topic_id => "zysco_realtime"
codec => line {
format => "%{message}"
}
}
}
./bin/logstash -f config/input-output.conf -t
需要多研究 grok
vim config/pipelines.yml
pipeline.id: id1
pipeline.workers: 1
path.config: "/home/software/logstash/config/input-output.conf"
20.3 自启动
nohup ./bin/logstash -f config/input-output.conf
vim /usr/lib/systemd/system/logstash.service
[Unit]
Description=logstash
After=network.target
[Service]
Type=simple
User=prometheus2
ExecStart=/home/software/logstash/bin/logstash -f /home/software/logstash/config/input-output.conf
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
systemctl daemon-reload
systemctl enable logstash.service
systemctl start logstash.service
systemctl status logstash.service
systemctl stop logstash.service
systemctl restart logstash.service
20.4 手动启动和停止
nohup /home/software/logstash/bin/logstash -f /home/software/logstash/config/input-output.conf
ps -aux|grep logstash
Kill pid
20.4 文件读取、清洗过滤
结晶器专家项目应用(window版本)
1.源文件
*.mes_caster
2.input + filter
input {
file{
path => "Z:/zhongyuan-ccm1_*/mex_cb_boExpert_*.mex_caster"
start_position => "beginning"
sincedb_path => "D:/logstash-7.3.2/data/plugins/inputs/file/dev/db-bo-expert"
type => "boExpert"
}
}
filter{
grok{
match => {
"message" => [
"%{WORD};%{NUMBER};%{NUMBER};%{NUMBER};;;%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{WORD};;%{NUMBER};%{NUMBER};",
"%{WORD};;%{NUMBER};;%{NUMBER};;%{NUMBER};;%{NUMBER};%{NUMBER};%{NUMBER};;%{NUMBER};%{NUMBER};%{WORD};%{WORD};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};;;%{NUMBER};%{NUMBER};%{NUMBER};%{WORD};;%{NUMBER};",
"%{WORD};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{NUMBER};%{WORD};%{NUMBER};%{NUMBER};%{WORD};%{NUMBER};%{NUMBER};;%{NUMBER};%{NUMBER};"
]
}
}
if[type] == "boExpert"{
mutate {
split => ["message",";"]
add_field => {"TIMESTAMP" => "%{[message][0]}"}
add_field => {"castingSpeed" => "%{[message][1]}"}
add_field => {"castingLength" => "%{[message][2]}"}
add_field => {"moldLevel" => "%{[message][3]}"}
add_field => {"moldLevelLimitLow" => "%{[message][4]}"}
add_field => {"moldLevelLimitUpp" => "%{[message][5]}"}
add_field => {"moldLevelDistance" => "%{[message][6]}"}
add_field => {"SENImmersionDepth" => "%{[message][7]}"}
add_field => {"stopper" => "%{[message][8]}"}
add_field => {"superHeat" => "%{[message][9]}"}
add_field => {"tundishTemperature" => "%{[message][10]}"}
add_field => {"castMode" => "%{[message][11]}"}
add_field => {"newSteelInMold" => "%{[message][12]}"}
add_field => {"moldConfiguration" => "%{[message][13]}"}
add_field => {"moldNumber" => "%{[message][14]}"}
add_field => {"bff_c01_1" => "%{[message][15]}"}
add_field => {"bff_c01_2" => "%{[message][16]}"}
add_field => {"bff_c01_3" => "%{[message][17]}"}
add_field => {"bff_c02_1" => "%{[message][18]}"}
add_field => {"bff_c02_2" => "%{[message][19]}"}
add_field => {"bff_c02_3" => "%{[message][20]}"}
add_field => {"bff_c03_1" => "%{[message][21]}"}
add_field => {"bff_c03_2" => "%{[message][22]}"}
add_field => {"bff_c03_3" => "%{[message][23]}"}
add_field => {"bff_c04_1" => "%{[message][24]}"}
add_field => {"bff_c04_2" => "%{[message][25]}"}
add_field => {"bff_c04_3" => "%{[message][26]}"}
add_field => {"bff_c05_1" => "%{[message][27]}"}
add_field => {"bff_c05_2" => "%{[message][28]}"}
add_field => {"bff_c05_3" => "%{[message][29]}"}
add_field => {"bff_c06_1" => "%{[message][30]}"}
add_field => {"bff_c06_2" => "%{[message][31]}"}
add_field => {"bff_c06_3" => "%{[message][32]}"}
add_field => {"bff_c07_1" => "%{[message][33]}"}
add_field => {"bff_c07_2" => "%{[message][34]}"}
add_field => {"bff_c07_3" => "%{[message][35]}"}
add_field => {"nfr_c01_1" => "%{[message][36]}"}
add_field => {"nfr_c01_2" => "%{[message][37]}"}
add_field => {"nfr_c01_3" => "%{[message][38]}"}
add_field => {"bfl_c01_1" => "%{[message][39]}"}
add_field => {"bfl_c01_2" => "%{[message][40]}"}
add_field => {"bfl_c01_3" => "%{[message][41]}"}
add_field => {"bfl_c02_1" => "%{[message][42]}"}
add_field => {"bfl_c02_2" => "%{[message][43]}"}
add_field => {"bfl_c02_3" => "%{[message][44]}"}
add_field => {"bfl_c03_1" => "%{[message][45]}"}
add_field => {"bfl_c03_2" => "%{[message][46]}"}
add_field => {"bfl_c03_3" => "%{[message][47]}"}
add_field => {"bfl_c04_1" => "%{[message][48]}"}
add_field => {"bfl_c04_2" => "%{[message][49]}"}
add_field => {"bfl_c04_3" => "%{[message][50]}"}
add_field => {"bfl_c05_1" => "%{[message][51]}"}
add_field => {"bfl_c05_2" => "%{[message][52]}"}
add_field => {"bfl_c05_3" => "%{[message][53]}"}
add_field => {"bfl_c06_1" => "%{[message][54]}"}
add_field => {"bfl_c06_2" => "%{[message][55]}"}
add_field => {"bfl_c06_3" => "%{[message][56]}"}
add_field => {"nfl_c01_1" => "%{[message][57]}"}
add_field => {"nfl_c01_2" => "%{[message][58]}"}
add_field => {"nfl_c01_3" => "%{[message][59]}"}
add_field => {"bff_disabled" => "%{[message][60]}"}
add_field => {"nfr_disabled" => "%{[message][61]}"}
add_field => {"bfl_disabled" => "%{[message][62]}"}
add_field => {"nfl_disabled" => "%{[message][63]}"}
add_field => {"castingSituation" => "%{[message][64]}"}
add_field => {"castingConditions" => "%{[message][65]}"}
add_field => {"BOP_alarms" => "%{[message][66]}"}
add_field => {"stopCaster" => "%{[message][67]}"}
add_field => {"BOP_strandPLCmode" => "%{[message][68]}"}
remove_field => ["message"]
}
if [TIMESTAMP] == "TIMESTAMP" {
mutate {remove_field => "TIMESTAMP"}
}
if [castingSpeed] == "castingSpeed" {
mutate {remove_field => "castingSpeed"}
}
if [castingLength] == "castingLength" {
mutate {remove_field => "castingLength"}
}
if [moldLevel] == "moldLevel" {
mutate {remove_field => "moldLevel"}
}
if [moldLevelLimitLow] == "moldLevelLimitLow" {
mutate {remove_field => "moldLevelLimitLow"}
}
if [moldLevelLimitUpp] == "moldLevelLimitUpp" {
mutate {remove_field => "moldLevelLimitUpp"}
}
if [moldLevelDistance] == "moldLevelDistance" {
mutate {remove_field => "moldLevelDistance"}
}
if [SENImmersionDepth] == "SENImmersionDepth" {
mutate {remove_field => "SENImmersionDepth"}
}
if [stopper] == "stopper" {
mutate {remove_field => "stopper"}
}
if [superHeat] == "superHeat" {
mutate {remove_field => "superHeat"}
}
if [tundishTemperature] == "tundishTemperature" {
mutate {remove_field => "tundishTemperature"}
}
if [castMode] == "castMode" {
mutate {remove_field => "castMode"}
}
if [newSteelInMold] == "newSteelInMold" {
mutate {remove_field => "newSteelInMold"}
}
if [moldConfiguration] == "moldConfiguration" {
mutate {remove_field => "moldConfiguration"}
}
if [moldNumber] == "moldNumber" {
mutate {remove_field => "moldNumber"}
}
if [bff_c01_1] == "bff_c01_1" {
mutate {remove_field => "bff_c01_1"}
}
if [bff_c01_2] == "bff_c01_2" {
mutate {remove_field => "bff_c01_2"}
}
if [bff_c01_3] == "bff_c01_3" {
mutate {remove_field => "bff_c01_3"}
}
if [bff_c02_1] == "bff_c02_1" {
mutate {remove_field => "bff_c02_1"}
}
if [bff_c02_2] == "bff_c02_2" {
mutate {remove_field => "bff_c02_2"}
}
if [bff_c02_3] == "bff_c02_3" {
mutate {remove_field => "bff_c02_3"}
}
if [bff_c03_1] == "bff_c03_1" {
mutate {remove_field => "bff_c03_1"}
}
if [bff_c03_2] == "bff_c03_2" {
mutate {remove_field => "bff_c03_2"}
}
if [bff_c03_3] == "bff_c03_3" {
mutate {remove_field => "bff_c03_3"}
}
if [bff_c04_1] == "bff_c04_1" {
mutate {remove_field => "bff_c04_1"}
}
if [bff_c04_2] == "bff_c04_2" {
mutate {remove_field => "bff_c04_2"}
}
if [bff_c04_3] == "bff_c04_3" {
mutate {remove_field => "bff_c04_3"}
}
if [bff_c05_1] == "bff_c05_1" {
mutate {remove_field => "bff_c05_1"}
}
if [bff_c05_2] == "bff_c05_2" {
mutate {remove_field => "bff_c05_2"}
}
if [bff_c05_3] == "bff_c05_3" {
mutate {remove_field => "bff_c05_3"}
}
if [bff_c06_1] == "bff_c06_1" {
mutate {remove_field => "bff_c06_1"}
}
if [bff_c06_2] == "bff_c06_2" {
mutate {remove_field => "bff_c06_2"}
}
if [bff_c06_3] == "bff_c06_3" {
mutate {remove_field => "bff_c06_3"}
}
if [bff_c07_1] == "bff_c07_1" {
mutate {remove_field => "bff_c07_1"}
}
if [bff_c07_2] == "bff_c07_2" {
mutate {remove_field => "bff_c07_2"}
}
if [bff_c07_3] == "bff_c07_3" {
mutate {remove_field => "bff_c07_3"}
}
if [nfr_c01_1] == "nfr_c01_1" {
mutate {remove_field => "nfr_c01_1"}
}
if [nfr_c01_2] == "nfr_c01_2" {
mutate {remove_field => "nfr_c01_2"}
}
if [nfr_c01_3] == "nfr_c01_3" {
mutate {remove_field => "nfr_c01_3"}
}
if [bfl_c01_1] == "bfl_c01_1" {
mutate {remove_field => "bfl_c01_1"}
}
if [bfl_c01_2] == "bfl_c01_2" {
mutate {remove_field => "bfl_c01_2"}
}
if [bfl_c01_3] == "bfl_c01_3" {
mutate {remove_field => "bfl_c01_3"}
}
if [bfl_c02_1] == "bfl_c02_1" {
mutate {remove_field => "bfl_c02_1"}
}
if [bfl_c02_2] == "bfl_c02_2" {
mutate {remove_field => "bfl_c02_2"}
}
if [bfl_c02_3] == "bfl_c02_3" {
mutate {remove_field => "bfl_c02_3"}
}
if [bfl_c03_1] == "bfl_c03_1" {
mutate {remove_field => "bfl_c03_1"}
}
if [bfl_c03_2] == "bfl_c03_2" {
mutate {remove_field => "bfl_c03_2"}
}
if [bfl_c03_3] == "bfl_c03_3" {
mutate {remove_field => "bfl_c03_3"}
}
if [bfl_c04_1] == "bfl_c04_1" {
mutate {remove_field => "bfl_c04_1"}
}
if [bfl_c04_2] == "bfl_c04_2" {
mutate {remove_field => "bfl_c04_2"}
}
if [bfl_c04_3] == "bfl_c04_3" {
mutate {remove_field => "bfl_c04_3"}
}
if [bfl_c05_1] == "bfl_c05_1" {
mutate {remove_field => "bfl_c05_1"}
}
if [bfl_c05_2] == "bfl_c05_2" {
mutate {remove_field => "bfl_c05_2"}
}
if [bfl_c05_3] == "bfl_c05_3" {
mutate {remove_field => "bfl_c05_3"}
}
if [bfl_c06_1] == "bfl_c06_1" {
mutate {remove_field => "bfl_c06_1"}
}
if [bfl_c06_2] == "bfl_c06_2" {
mutate {remove_field => "bfl_c06_2"}
}
if [bfl_c06_3] == "bfl_c06_3" {
mutate {remove_field => "bfl_c06_3"}
}
if [nfl_c01_1] == "nfl_c01_1" {
mutate {remove_field => "nfl_c01_1"}
}
if [nfl_c01_2] == "nfl_c01_2" {
mutate {remove_field => "nfl_c01_2"}
}
if [nfl_c01_3] == "nfl_c01_3" {
mutate {remove_field => "nfl_c01_3"}
}
if [bff_disabled] == "bff_disabled" {
mutate {remove_field => "bff_disabled"}
}
if [nfr_disabled] == "nfr_disabled" {
mutate {remove_field => "nfr_disabled"}
}
if [bfl_disabled] == "bfl_disabled" {
mutate {remove_field => "bfl_disabled"}
}
if [nfl_disabled] == "nfl_disabled" {
mutate {remove_field => "nfl_disabled"}
}
if [castingSituation] == "castingSituation" {
mutate {remove_field => "castingSituation"}
}
if [castingConditions] == "castingConditions" {
mutate {remove_field => "castingConditions"}
}
if [BOP_alarms] == "BOP_alarms" {
mutate {remove_field => "BOP_alarms"}
}
if [stopCaster] == "stopCaster" {
mutate {remove_field => "stopCaster"}
}
if [BOP_strandPLCmode] == "BOP_strandPLCmode" {
mutate {remove_field => "BOP_strandPLCmode"}
}
if [TIMESTAMP] == "%{[message][0]}" {
mutate {remove_field => "TIMESTAMP"}
}
if [castingSpeed] == "%{[message][1]}" {
mutate {remove_field => "castingSpeed"}
}
if [castingLength] == "%{[message][2]}" {
mutate {remove_field => "castingLength"}
}
if [moldLevel] == "%{[message][3]}" {
mutate {remove_field => "moldLevel"}
}
if [moldLevelLimitLow] == "%{[message][4]}" {
mutate {remove_field => "moldLevelLimitLow"}
}
if [moldLevelLimitUpp] == "%{[message][5]}" {
mutate {remove_field => "moldLevelLimitUpp"}
}
if [moldLevelDistance] == "%{[message][6]}" {
mutate {remove_field => "moldLevelDistance"}
}
if [SENImmersionDepth] == "%{[message][7]}" {
mutate {remove_field => "SENImmersionDepth"}
}
if [stopper] == "%{[message][8]}" {
mutate {remove_field => "stopper"}
}
if [superHeat] == "%{[message][9]}" {
mutate {remove_field => "superHeat"}
}
if [tundishTemperature] == "%{[message][10]}" {
mutate {remove_field => "tundishTemperature"}
}
if [castMode] == "%{[message][11]}" {
mutate {remove_field => "castMode"}
}
if [newSteelInMold] == "%{[message][12]}" {
mutate {remove_field => "newSteelInMold"}
}
if [moldConfiguration] == "%{[message][13]}" {
mutate {remove_field => "moldConfiguration"}
}
if [moldNumber] == "%{[message][14]}" {
mutate {remove_field => "moldNumber"}
}
if [bff_c01_1] == "%{[message][15]}" {
mutate {remove_field => "bff_c01_1"}
}
if [bff_c01_2] == "%{[message][16]}" {
mutate {remove_field => "bff_c01_2"}
}
if [bff_c01_3] == "%{[message][17]}" {
mutate {remove_field => "bff_c01_3"}
}
if [bff_c02_1] == "%{[message][18]}" {
mutate {remove_field => "bff_c02_1"}
}
if [bff_c02_2] == "%{[message][19]}" {
mutate {remove_field => "bff_c02_2"}
}
if [bff_c02_3] == "%{[message][20]}" {
mutate {remove_field => "bff_c02_3"}
}
if [bff_c03_1] == "%{[message][21]}" {
mutate {remove_field => "bff_c03_1"}
}
if [bff_c03_2] == "%{[message][22]}" {
mutate {remove_field => "bff_c03_2"}
}
if [bff_c03_3] == "%{[message][23]}" {
mutate {remove_field => "bff_c03_3"}
}
if [bff_c04_1] == "%{[message][24]}" {
mutate {remove_field => "bff_c04_1"}
}
if [bff_c04_2] == "%{[message][25]}" {
mutate {remove_field => "bff_c04_2"}
}
if [bff_c04_3] == "%{[message][26]}" {
mutate {remove_field => "bff_c04_3"}
}
if [bff_c05_1] == "%{[message][27]}" {
mutate {remove_field => "bff_c05_1"}
}
if [bff_c05_2] == "%{[message][28]}" {
mutate {remove_field => "bff_c05_2"}
}
if [bff_c05_3] == "%{[message][29]}" {
mutate {remove_field => "bff_c05_3"}
}
if [bff_c06_1] == "%{[message][30]}" {
mutate {remove_field => "bff_c06_1"}
}
if [bff_c06_2] == "%{[message][31]}" {
mutate {remove_field => "bff_c06_2"}
}
if [bff_c06_3] == "%{[message][32]}" {
mutate {remove_field => "bff_c06_3"}
}
if [bff_c07_1] == "%{[message][33]}" {
mutate {remove_field => "bff_c07_1"}
}
if [bff_c07_2] == "%{[message][34]}" {
mutate {remove_field => "bff_c07_2"}
}
if [bff_c07_3] == "%{[message][35]}" {
mutate {remove_field => "bff_c07_3"}
}
if [nfr_c01_1] == "%{[message][36]}" {
mutate {remove_field => "nfr_c01_1"}
}
if [nfr_c01_2] == "%{[message][37]}" {
mutate {remove_field => "nfr_c01_2"}
}
if [nfr_c01_3] == "%{[message][38]}" {
mutate {remove_field => "nfr_c01_3"}
}
if [bfl_c01_1] == "%{[message][39]}" {
mutate {remove_field => "bfl_c01_1"}
}
if [bfl_c01_2] == "%{[message][40]}" {
mutate {remove_field => "bfl_c01_2"}
}
if [bfl_c01_3] == "%{[message][41]}" {
mutate {remove_field => "bfl_c01_3"}
}
if [bfl_c02_1] == "%{[message][42]}" {
mutate {remove_field => "bfl_c02_1"}
}
if [bfl_c02_2] == "%{[message][43]}" {
mutate {remove_field => "bfl_c02_2"}
}
if [bfl_c02_3] == "%{[message][44]}" {
mutate {remove_field => "bfl_c02_3"}
}
if [bfl_c03_1] == "%{[message][45]}" {
mutate {remove_field => "bfl_c03_1"}
}
if [bfl_c03_2] == "%{[message][46]}" {
mutate {remove_field => "bfl_c03_2"}
}
if [bfl_c03_3] == "%{[message][47]}" {
mutate {remove_field => "bfl_c03_3"}
}
if [bfl_c04_1] == "%{[message][48]}" {
mutate {remove_field => "bfl_c04_1"}
}
if [bfl_c04_2] == "%{[message][49]}" {
mutate {remove_field => "bfl_c04_2"}
}
if [bfl_c04_3] == "%{[message][50]}" {
mutate {remove_field => "bfl_c04_3"}
}
if [bfl_c05_1] == "%{[message][51]}" {
mutate {remove_field => "bfl_c05_1"}
}
if [bfl_c05_2] == "%{[message][52]}" {
mutate {remove_field => "bfl_c05_2"}
}
if [bfl_c05_3] == "%{[message][53]}" {
mutate {remove_field => "bfl_c05_3"}
}
if [bfl_c06_1] == "%{[message][54]}" {
mutate {remove_field => "bfl_c06_1"}
}
if [bfl_c06_2] == "%{[message][55]}" {
mutate {remove_field => "bfl_c06_2"}
}
if [bfl_c06_3] == "%{[message][56]}" {
mutate {remove_field => "bfl_c06_3"}
}
if [nfl_c01_1] == "%{[message][57]}" {
mutate {remove_field => "nfl_c01_1"}
}
if [nfl_c01_2] == "%{[message][58]}" {
mutate {remove_field => "nfl_c01_2"}
}
if [nfl_c01_3] == "%{[message][59]}" {
mutate {remove_field => "nfl_c01_3"}
}
if [bff_disabled] == "%{[message][60]}" {
mutate {remove_field => "bff_disabled"}
}
if [nfr_disabled] == "%{[message][61]}" {
mutate {remove_field => "nfr_disabled"}
}
if [bfl_disabled] == "%{[message][62]}" {
mutate {remove_field => "bfl_disabled"}
}
if [nfl_disabled] == "%{[message][63]}" {
mutate {remove_field => "nfl_disabled"}
}
if [castingSituation] == "%{[message][64]}" {
mutate {remove_field => "castingSituation"}
}
if [castingConditions] == "%{[message][65]}" {
mutate {remove_field => "castingConditions"}
}
if [BOP_alarms] == "%{[message][66]}" {
mutate {remove_field => "BOP_alarms"}
}
if [stopCaster] == "%{[message][67]}" {
mutate {remove_field => "stopCaster"}
}
if [BOP_strandPLCmode] == "%{[message][68]}" {
mutate {remove_field => "BOP_strandPLCmode"}
}
}
}
3.output
output {
stdout{
codec => rubydebug
}
mqtt{
host => "172.16.10.5"
port => 1883
topic => "test"
client_id => "starquest"
username => "starquest"
password => "starquest"
}
}
4.启动
logstash.bat -f mex
mex 为当前文件夹下的配置文件
多实例启动
logstash.bat -f mex2 --path.data=/path/data
5.重要的文件目录
配置文件目录 D:\logstash-7.3.2\bin\mex
记录已读取地址 D:\logstash-7.3.2\data\plugins\inputs\file\dev
日志路径 D:\logstash-7.3.2\logs