环境：centos7.6 、graylog3版本
架构：前端负载采用nginx、后端node分别部署graylog3集群、ES6集群、mongodb集群。
Graylog整体组成：
Graylog：提供 graylog 对外接口，web UI， CPU
Elasticsearch：日志文件的持久化存储和检索， IO
MongoDB：存储一些 Graylog 的配置 数据
Nginx：负载均衡
逻辑架构.png
开始部署：
1、关闭防火墙及selinux
#主机23.231-234同步操作：
[root@graylog01 ~]# sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
[root@graylog01 ~]# setenforce 0
[root@graylog01 ~]# systemctl stop firewalld
[root@graylog01 ~]# systemctl disable firewalld
#修改主机名：
[root@graylog01 ~]# hostnamectl set-hostname graylog01
2、安装依赖
//主机23.232-234同步操作：
[root@graylog01 ~]# yum install epel-release pwgen java-1.8.0-openjdk-headless.x86_64 -y
3、安装MongoDB
#主机23.232-234同步操作：
[root@graylog01 ~]# touch /etc/yum.repos.d/mongodb-org.repo
[root@graylog01 ~]# cat << EOF > /etc/yum.repos.d/mongodb-org.repo
[mongodb-org-4.0]
name=MongoDB Repository
baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.0/x86_64/
gpgcheck=1
enabled=1
gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc
EOF
[root@graylog01 ~]# yum install -y mongodb-org
[root@graylog01 ~]# systemctl daemon-reload
[root@graylog01 ~]# systemctl enable mongod.service
[root@graylog01 ~]# systemctl start mongod.service
#运行
[root@graylog01 ~]# mongo --host 127.0.0.1:27017
> db
> show dbs
#配置远程连接
#编辑 mongod.conf 配置
[root@graylog01 ~]# cp /etc/mongod.conf /etc/mongod.conf.bak
[root@graylog01 ~]# vi /etc/mongod.conf
#将 bindIp: 127.0.0.1 改成 bindIp: 0.0.0.0，这里注意 tab 和空格对齐
#开放端口
[root@graylog01 ~]# firewall-cmd --zone=public --add-port=27017/tcp --permanent
[root@graylog01 ~]# firewall-cmd --reload
[root@graylog01 ~]# firewall-cmd --zone=public --query-port=27017/tcp
#重启服务
[root@graylog01 ~]# systemctl restart mongod.service
#【集群配置】
#修改所有mongdb节点的配置文件/etc/mongod.conf，添加集群replication信息replSetName: rs0，并重启服务。
#配置如下：
[root@graylog01 ~]# cat /etc/mongod.conf |grep -Ev "^#|^$"
systemLog:
destination: file
logAppend: true
path: /data/mongo/log/mongod.log
#path: /var/log/mongodb/mongod.log
storage:
dbPath: /data/mongo/data
#dbPath: /var/lib/mongo
journal:
enabled: true
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongodb/mongod.pid # location of pidfile
timeZoneInfo: /usr/share/zoneinfo
net:
port: 27017
bindIp: 0.0.0.0 # Enter 0.0.0.0,:: to bind to all IPv4 and IPv6 addresses or, alternatively, use the net.bindIpAll setting.
replication:
replSetName: rs0
#之后，重启数据库
systemctl restart mongod.service
#初始化mongodb集群（在其中一台操作），使用本机hostname或IP加端口，当前操作主机172.16.23.231：
[root@graylog01 ~]# mongo
> rs.initiate( {
_id : "rs0",
members: [ { _id : 0, host : "172.16.23.232:27017" } ]
})
#验证集群配置
rs0:PRIMARY> rs.conf()
{
"_id" : "rs0",
"version" : 3,
"protocolVersion" : NumberLong(1),
"members" : [
{
"_id" : 0,
"host" : "172.16.23.232:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 1,
"tags" : {
},
"slaveDelay" : NumberLong(0),
"votes" : 1
}
],
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : 2000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("5e182b012ef13614f3b11681")
}
}
#将其他节点加入集群，并查看集群配置：
rs0:PRIMARY> rs.add("172.16.23.233")
rs0:PRIMARY> rs.add("172.16.23.234")
rs0:PRIMARY> rs.status()
{
"set" : "rs0",
"date" : ISODate("2020-03-28T07:27:08.144Z"),
"myState" : 1,
"term" : NumberLong(11),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1585380427, 8),
"t" : NumberLong(11)
},
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1585380427, 8),
"t" : NumberLong(11)
},
"appliedOpTime" : {
"ts" : Timestamp(1585380428, 1),
"t" : NumberLong(11)
},
"durableOpTime" : {
"ts" : Timestamp(1585380427, 8),
"t" : NumberLong(11)
}
},
"lastStableCheckpointTimestamp" : Timestamp(1585380393, 5),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2020-03-21T13:29:36.101Z"),
"electionTerm" : NumberLong(11),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(1584795559, 2),
"t" : NumberLong(10)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1584795569, 1),
"t" : NumberLong(10)
},
"numVotesNeeded" : 2,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"numCatchUpOps" : NumberLong(0),
"newTermStartDate" : ISODate("2020-03-21T13:29:37.607Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2020-03-21T13:29:37.772Z")
},
"members" : [
{
"_id" : 0,
"name" : "172.16.23.232:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 4831953,
"optime" : {
"ts" : Timestamp(1585380428, 1),
"t" : NumberLong(11)
},
"optimeDate" : ISODate("2020-03-28T07:27:08Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1584797376, 1),
"electionDate" : ISODate("2020-03-21T13:29:36Z"),
"configVersion" : 3,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "172.16.23.233:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 583041,
"optime" : {
"ts" : Timestamp(1585380426, 2),
"t" : NumberLong(11)
},
"optimeDurable" : {
"ts" : Timestamp(1585380426, 2),
"t" : NumberLong(11)
},
"optimeDate" : ISODate("2020-03-28T07:27:06Z"),
"optimeDurableDate" : ISODate("2020-03-28T07:27:06Z"),
"lastHeartbeat" : ISODate("2020-03-28T07:27:07.295Z"),
"lastHeartbeatRecv" : ISODate("2020-03-28T07:27:06.814Z"),
"pingMs" : NumberLong(4),
"lastHeartbeatMessage" : "",
"syncingTo" : "172.16.23.232:27017",
"syncSourceHost" : "172.16.23.232:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 3
},
{
"_id" : 2,
"name" : "172.16.23.234:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 583054,
"optime" : {
"ts" : Timestamp(1585380426, 1),
"t" : NumberLong(11)
},
"optimeDurable" : {
"ts" : Timestamp(1585380426, 1),
"t" : NumberLong(11)
},
"optimeDate" : ISODate("2020-03-28T07:27:06Z"),
"optimeDurableDate" : ISODate("2020-03-28T07:27:06Z"),
"lastHeartbeat" : ISODate("2020-03-28T07:27:06.826Z"),
"lastHeartbeatRecv" : ISODate("2020-03-28T07:27:08.065Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "172.16.23.232:27017",
"syncSourceHost" : "172.16.23.232:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 3
}
],
"ok" : 1,
"operationTime" : Timestamp(1585380428, 1),
"$clusterTime" : {
"clusterTime" : Timestamp(1585380428, 1),
"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
"keyId" : NumberLong(0)
}
}
}
#创建graylog数据库，并添加graylog用户，赋予readWrite和dbAdmin权限：
rs0:PRIMARY> use graylog
switched to db graylog
rs0:PRIMARY> db.createUser( {
user: "graylog",
pwd: "xxxxx",
roles: [ { role: "readWrite", db: "graylog" } ]
});
rs0:PRIMARY> db.grantRolesToUser( "graylog" , [ { role: "dbAdmin", db: "graylog" } ])
rs0:PRIMARY> show users
rs0:PRIMARY> db.auth("graylog","xxxxx")
4、Elasticsearch集群部署
# graylog3.0 使用的elasticsearch不低于5.6.13版本，我这里用的最6.4.3（暂不支持7.x，亲测）
主机23.232-234同步操作：
编译安装软件包：
[root@graylog01 ~]# wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.4.3.tar.gz
[root@graylog01 ~]# tar -zxf elasticsearch-6.4.3.tar.gz
[root@graylog01 ~]# mv elasticsearch-6.4.3 /usr/local/elasticsearch
[root@graylog01 ~]# mkdir -p /usr/local/elasticsearch/data #创建数据目录
[root@graylog01 ~]# mkdir -p /usr/local/elasticsearch/logs #创建日志目录
#elasticsearch不能用root用户启动
[root@graylog01 ~]# useradd elk
[root@graylog01 ~]# chown -R elk:elk /usr/local/elasticsearch #赋予权限
[root@graylog01 ~]# vim /etc/profile #添加环境变量
export JAVA_HOME=/usr/local/jdk
export CLASSPATH=$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin
[root@graylog01 ~]# source /etc/profile #环境变量立即生效
[root@graylog01 ~]# java -version #查看java版本（本机jdk源码安装）
[root@graylog01 ~]# java -version
java version "1.8.0_181"
Java(TM) SE Runtime Environment (build 1.8.0_181-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.181-b13, mixed mode)
#配置信息：
[root@graylog01 ~]# cat /usr/local/elasticsearch/config/elasticsearch.yml |grep -Ev "^#|^$"
#graylog01：
cluster.name: graylog3
node.name: graylog01
node.master: true
node.data: true
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["172.16.23.232:9300", "172.16.23.233:9300","172.16.23.234:9300"]
#graylog02
cluster.name: graylog3
node.name: graylog02
node.master: true
node.data: true
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["172.16.23.232:9300", "172.16.23.233:9300", "172.16.23.234:9300"]
#graylog03
cluster.name: graylog3
node.name: graylog03
node.master: true
node.data: true
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/logs
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["172.16.23.232:9300", "172.16.23.233:9300", "172.16.23.234:9300"]
#启动Elasticsearch服务：
[编译安装启动]
[root@graylog01 ~]# su - elk
[root@graylog01 ~]# /data/elasticsearch/bin/elasticsearch -d # -d 后台运行模式
[yum安装启动]
[root@graylog01 ~]# chkconfig --add elasticsearch
[root@graylog01 ~]# systemctl daemon-reload
[root@graylog01 ~]# systemctl enable elasticsearch.service
[root@graylog01 ~]# systemctl restart elasticsearch.service
5、Graylog安装
[root@graylog01 ~]# rpm -Uvh https://packages.graylog2.org/repo/packages/graylog-3.0-repository_latest.rpm
[root@graylog01 ~]# yum install graylog-server -y
#修改配置文件：， password_secret和root_password_sha2是必须的，不设置则无法启动，设置方法如下：
[root@graylog01 ~]# vim /etc/graylog/server/server.conf
---------------------------------------------------------------------------------
# passworde_secret可以通过命令：pwgen -N 1 -s 96 来随机生成，下面就是我随机生成的
password_secret = PLb50jbamnyu3QdWH0o9vYxytEvAH7I6OCjIGxEcKJO08ZHmbYMi00vh1GdJYn5NBItg86hr58imlIUzdE8elVzKkML1oC2Y
# admin用户密码生成命令：echo -n yourpassword | sha256sum
# 生成后，请记住你的 YourPassword(三台一致)
root_password_sha2 = ac9689e2272427085e35b9d3e3e8bed88cb3434828b43b86fc0596cad4c6e270
# admin用户邮箱
root_email = "root@example.com"
# 时区
root_timezone = Asia/Shanghai
# 检索高亮
allow_highlighting = true
# elasticsearch 相关配置
elasticsearch_hosts = http://graylog01:9200,http://graylog02:9200,http://graylog03:9200 #集群模式
elasticsearch_shards =1
elasticsearch_replicas = 0
# mongodb 连接配置，这里直接本机起的mongodb，没有设置验证
mongodb_uri = mongodb://localhost/graylog
# 电子邮件smtp,设置为自己的邮箱smtp服务
transport_email_enabled = true
transport_email_hostname = smtp.exmail.qq.com
transport_email_port = 465
transport_email_use_auth = true
transport_email_use_tls = false
transport_email_use_ssl = true
transport_email_auth_username = root@example.com
transport_email_auth_password = 123456
transport_email_subject_prefix = [graylog]
transport_email_from_email = root@example.com
transport_email_web_interface_url = http://graylog.example.com
# 网络访问相关，重要，graylog3比2.x版本简洁了很多网络配置，只需配置http_bind_address即可。
http_bind_address = 0.0.0.0:9000
# 配置外网地址，我这里用了域名+nginx做反向代理，所以外网地址如下。没有的话就直接就用外网ip+port，如：http://外网ip:9000/
http_publish_uri = http://172.16.23.232:9000/
# http_external_uri = http://graylog.example.com/ 单节点的话，此配置不需要配置，默认使用http_publish_uri
---------------------------------------------------------------------------------
#线上环境配置文件参考
[root@graylog01 data]# grep -Ev "^#|^$" /etc/graylog/server/server.conf
is_master = true
node_id_file = /etc/graylog/server/node-id
password_secret = 7N84Dhanhy0bFwLscsDXcuWFxdFYg5ZBpYUEdIGR
root_password_sha2 = ac9689e2272427085e35b9d3e3e8bed88cb3434828b43b86fc0596cad4c6e270
root_timezone = Asia/Shanghai
bin_dir = /usr/share/graylog-server/bin
data_dir = /var/lib/graylog-server
plugin_dir = /usr/share/graylog-server/plugin
http_bind_address = 0.0.0.0:9000
http_publish_uri = http://172.16.23.232:9000/
elasticsearch_hosts = http://graylog01:9200,http://graylog02:9200,http://graylog03:9200
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_number_of_indices = 20
retention_strategy = delete
elasticsearch_shards = 4
elasticsearch_replicas = 0
elasticsearch_index_prefix = graylog
allow_leading_wildcard_searches = false
allow_highlighting = true
elasticsearch_analyzer = standard
output_batch_size = 500
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 3
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
message_journal_enabled = true
message_journal_dir = /var/lib/graylog-server/journal
lb_recognition_period_seconds = 3
mongodb_uri = mongodb://172.16.23.232:27017,172.16.23.233:27017,172.16.23.234:27017/graylog
mongodb_max_connections = 1000
mongodb_threads_allowed_to_block_multiplier = 5
proxied_requests_thread_pool_size = 32
---------------------------------------------------------------------------------
# 启动需要手动设置Java路径
[root@graylog01 ~]# vim /etc/sysconfig/graylog-server
---------------------------------------------------------------------------------
JAVA=/usr/local/jdk1.8/bin/java #根据实际路径填写即可
---------------------------------------------------------------------------------
# 启动服务
[root@graylog01 ~]# chkconfig --add graylog-server
[root@graylog01 ~]# systemctl daemon-reload
[root@graylog01 ~]# systemctl enable graylog-server.service
[root@graylog01 ~]# systemctl start graylog-server.service
6、web登录
#直接访问本机IP：172.16.23.232:9000
Web UI.png
7、安装Graylog Sidecar（Graylog Collector Sidecar）
Graylog Sidecar是一个轻量级配置管理系统，适用于不同的日志收集器，也称为后端。Graylog节点充当包含日志收集器配置的集中式集线器。在支持的消息生成设备/主机上，Sidecar可以作为服务（Windows主机）或守护程序（Linux主机）运行。进行在不同机器上进行日志的采集并发送到graylog server.在graylog3.0版本以前，称为Graylog Collector Sidecar，在3.0中改为了Graylog Sidecar，在官方文档中有详细安装指导：官方文档入口(http://docs.graylog.org/en/3.0/pages/sidecar.html)。这里也参考进行安装。版本对照表如下，首先去github上下载相应的rpm安装包。官方GITHUB下载地址(https://github.com/Graylog2/collector-sidecar/releases)
逻辑图.png
Sidecar version Graylog server version
| Sidecar version | Graylog server version |
|---|---|
| 1.0.x | 3.0.x |
| 0.1.x | 2.2.x,2.3.x,2.4.x,2.5.x,3.0.x |
| 0.0.9 | 2.1.x |
# 安装
[root@graylog01 ~]# rpm -i graylog-sidecar-1.0.2-1.x86_64.rpm
# 修改配置
[root@graylog01 ~]# grep -Ev "^#|^$" /etc/graylog/sidecar/sidecar.yml
----------------------------------------------------------------------------
server_url: "http://172.16.23.231:9000/api/" # api的外网地址，23.231为前端nginx负载地址，下文详细说明。
# api token 必要的，不然启动不了，token需要在web界面上进行手动创建
server_api_token: "cvm841q4odgofktsihn2t0jsuit0ulivdu5hpj8mh1vjph78t0e" #收集多台主机日志，这里可配置统一的api token，分业务区分，方便管理。
node_name: "graylog01" # 自定义节点名称
update_interval: 10
send_status: true
----------------------------------------------------------------------------
# 安装系统服务
[root@graylog01 ~]# graylog-sidecar -service install
[root@graylog01 ~]# systemctl start graylog-sidecar
#手动创建server_api_token。
创建server_api_token01.png
创建server_api_token02.png
现在可以启动graylog-sidecar了。启动后，在web界面上就可以看到一个节点了，然后下面记录怎么手动配置这个节点的日志采集。然后就需要定义sidecar的filebeat配置，用这个配置来启动filebeat进行日志采集，并输入到上面定义的beats input。但是graylog3.0中，graylog sidecar的linux版本不包含filebeat(3.0版本之前是默认包含filebeat的)，需要自己手动下载安装filebeat，安装非常简单，通过官方下载页面，直接下载rpm包进行安装就行：[https://www.elastic.co/cn/downloads/beats/filebeat]）PS：我这里是演示的用filebeat进行日志采集，如果用nxlog进行采集，同样的需要安装nxlog程序。
#安装filebeat
[root@graylog01 ~]# rpm -i filebeat-6.6.0-x86_64.rpm
8、nginx负载部署
引用快速部署：centos7 yum安装nginx
#添加配置
#web负载配置
upstream graylog_servers {
least_conn;
server 172.16.23.232:9000;
server 172.16.23.233:9000;
server 172.16.23.234:9000;
}
server {
listen 9000;
proxy_buffer_size 16k;
proxy_timeout 10s;
error_log graylog_servers.log;
proxy_pass graylog_servers;
}
9、graylog日志收集实践
客户端需要配置好graylog-sidecar及filebeat并且成功与server通信。
客户端成功监听.png
定义收集器.png
定义监听端口.png
配置收集策略.png
策略详细配置23.231为nginx负载端口.png
应用策略.png
#最后nginx添加配置
upstream graylog_server_log {
least_conn;
server 172.16.23.232:6004;
server 172.16.23.233:6004;
server 172.16.23.234:6004;
}
server {
listen 6004;
proxy_buffer_size 16k;
proxy_timeout 10s;
error_log graylog_servers.log;
proxy_pass graylog_server_log;
}
web搜索展示：
preview.png
为了便于条理化维护管理，我们可以对创建的日志进行分类管理，在实际工作中，服务日志会非常多，这么多的日志，如果不进行存储限制，那么不久就会占满磁盘，查询变慢等等，而且过久的历史日志对于实际工作中的有效性也会很低，graylog则自身集成了日志数据限制的配置，可以通过如下进行设置：：
创建索引.png
索引详细.png
详细配置里对日志的存储进行限制：
1、限定日志数据量，数据量达到是便会自动删除最旧的历史数据，以维持数据量恒定
2、限定大小，指定数据文件总容量大小
3、限定时间，删除超时的日志数据。 可根据自己的实际情况进行设置。
定义好索引，还可以基于索引创建stream流，对不同的日志进行分类管理：
stream.png
设置stream规则.png
10、可视化
1、创建dashboard
2、检索数据
3、将结果加入到dashboard
创建dashboard.png
检索数据.png
效果图.png
The End
#注：本文在作者[原文链接：https://blog.csdn.net/weixin_41004350/article/details/87253316及https://www.cnblogs.com/wayneiscoming/p/7635357.html]基础上进行归纳整理，并部署应用于线上系统，亲自实践应用。侵删。
