wget https://artifacts.elastic.co/downloads/logstash/logstash-7.1.0.tar.gz
tar -zxvf logstash-7.1.0.tar.gz
下载测试数据
wget http://files.grouplens.org/datasets/movielens/ml-20m.zip
unzip ml-20m.zip
编辑.conf文件
cd /opt/logstash-7.1.0/config
vim logstash.conf
input {
file {
path => "/opt/data/ml-20m/movies.csv" #编辑为你自己数据存储目录
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
csv {
separator => ","
columns => ["id","content","genre"]
}
mutate {
split => { "genre" => "|" }
remove_field => ["path", "host","@timestamp","message"]
}
mutate {
split => ["content", "("]
add_field => { "title" => "%{[content][0]}"}
add_field => { "year" => "%{[content][1]}"}
}
# mutate {
# gsub => [
#
# "year", "\\)", ""
# ]
# }
mutate {
convert => {
"year" => "integer"
}
strip => ["title"]
remove_field => ["path", "host","@timestamp","message","content"]
}
}
output {
elasticsearch {
hosts => "http://es-003:9200"# 编辑为你自己的es集群地址
index => "movies"
document_id => "%{id}"
}
stdout {}
}
导入数据到ES
#启动Elasticsearch实例,然后启动 logstash,并制定配置文件导入数据
bin/logstash -f /opt/logstash-7.1.0/config/logstash.conf