Windows下安装ES
进入到ES官网的下载页面：
https://www.elastic.co/cn/downloads/elasticsearch
然后点击下载，选择版本，我这里选择了最新版本：
image.png
然后在Windows下解压，直接到解压后的bin目录中，执行下面命令启动：
D:\bigdata\elasticsearch-7.15.0\bin>elasticsearch
然后出现下面信息表示启动成功：
[2021-09-24T09:43:44,940][INFO ][o.e.h.AbstractHttpServerTransport] [JTYSL-27LYMT2] publish_address {127.0.0.1:9200}, bound_addresses {127.0.0.1:9200}, {[::1]:9200}
[2021-09-24T09:43:44,941][INFO ][o.e.n.Node ] [JTYSL-27LYMT2] started
接下来在浏览器中访问 http://localhost:9200/,得到看到下面的结果，说明整个ES已经启动可用
image.png
Windows下安装Kibana
进入到ES官网的下载页面：
https://www.elastic.co/cn/downloads/kibana
然后点击下载，选择版本，我这里选择了最新版本：
image.png
然后在Windows下解压，直接到解压后的bin目录中，执行下面命令启动：
D:\bigdata\kibana-7.15.0-windows-x86_64\bin\kibana.bat
接下来在浏览器中访问 http://localhost:5601/,得到看到下面的结果，说明整个ES已经启动可用
image.png
ES基础操作
一个简单的查询
ES使用REST API 的方式对接提供查询接口，一个最简单的查询实例：
C:\Users\shikenian>curl -X GET http://localhost:9200/
{
"name" : "JTYSL-27LYMT2",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "EmJN6HdSTQuFvAzoMnSDRw",
"version" : {
"number" : "7.15.0",
"build_flavor" : "default",
"build_type" : "zip",
"build_hash" : "79d65f6e357953a5b3cbcc5e2c7c21073d89aa29",
"build_date" : "2021-09-16T03:05:29.143308416Z",
"build_snapshot" : false,
"lucene_version" : "8.9.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
添加数据
通过POST添加数据到ES的简单样例（Windows下CURL不是很好用，我用的是POSTMAN）
POST URL
localhost:9200/logs-my_app-default/_doc?pretty
JSON参数
{
"@timestamp": "2099-05-06T16:21:15.000Z",
"event": {
"original": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] GET /images/bg.jpg HTTP/1.0 200 24736"
}
}
返回值：
{
"_index": ".ds-logs-my_app-default-2021.09.24-000001",
"_type": "_doc",
"_id": "sHr7FXwBEPmjP8ocrxhf",
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"_seq_no": 0,
"_primary_term": 1
}
批量插入数据
HTTP类型：PUT
URL: localhost:9200/logs-my_app-default/_bulk
参数：
{ "create": { } }
{ "@timestamp": "2099-05-07T16:24:32.000Z", "event": { "original": "192.0.2.242 - - [07/May/2020:16:24:32 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" } }
{ "create": { } }
{ "@timestamp": "2099-05-08T16:25:42.000Z", "event": { "original": "192.0.2.255 - - [08/May/2099:16:25:42 +0000] \"GET /favicon.ico HTTP/1.0\" 200 3638" } }
结果：
{
"took": 17,
"errors": false,
"items": [
{
"create": {
"_index": ".ds-logs-my_app-default-2021.09.24-000001",
"_type": "_doc",
"_id": "t3oMFnwBEPmjP8ocCRiq",
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"_seq_no": 1,
"_primary_term": 1,
"status": 201
}
},
{
"create": {
"_index": ".ds-logs-my_app-default-2021.09.24-000001",
"_type": "_doc",
"_id": "uHoMFnwBEPmjP8ocCRiq",
"_version": 1,
"result": "created",
"_shards": {
"total": 2,
"successful": 1,
"failed": 0
},
"_seq_no": 2,
"_primary_term": 1,
"status": 201
}
}
]
}
查询数据
接下来是应用Kibina的DEV TOOLS 来操作。
- 查询所有的数据
GET logs-my_app-default/_search
{
"query": {
"match_all": { }
},
"sort": [
{
"@timestamp": "desc"
}
]
}
结果：
{
"took" : 3,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 3,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "uHoMFnwBEPmjP8ocCRiq",
"_score" : null,
"_source" : {
"@timestamp" : "2099-05-08T16:25:42.000Z",
"event" : {
"original" : """192.0.2.255 - - [08/May/2099:16:25:42 +0000] "GET /favicon.ico HTTP/1.0" 200 3638"""
}
},
"sort" : [
4081940742000
]
},
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "t3oMFnwBEPmjP8ocCRiq",
"_score" : null,
"_source" : {
"@timestamp" : "2099-05-07T16:24:32.000Z",
"event" : {
"original" : """192.0.2.242 - - [07/May/2020:16:24:32 -0500] "GET /images/hm_nbg.jpg HTTP/1.0" 304 0"""
}
},
"sort" : [
4081854272000
]
},
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "sHr7FXwBEPmjP8ocrxhf",
"_score" : null,
"_source" : {
"@timestamp" : "2099-05-06T16:21:15.000Z",
"event" : {
"original" : "192.0.2.42 - - [06/May/2099:16:21:15 +0000] GET /images/bg.jpg HTTP/1.0 200 24736"
}
},
"sort" : [
4081767675000
]
}
]
}
}
- 指定查询某个列，不查询所有列
查询：
指定查询 @timestamp 字段
不展示原始JSON文档，也就是排除 _source
GET logs-my_app-default/_search
{
"query": {
"match_all": { }
},
"_source": false,
"fields": [
"@timestamp"
],
"sort": [
{
"@timestamp": "desc"
}
]
}
结果如下：
{
"took" : 7,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 3,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "uHoMFnwBEPmjP8ocCRiq",
"_score" : null,
"fields" : {
"@timestamp" : [
"2099-05-08T16:25:42.000Z"
]
},
"sort" : [
4081940742000
]
},
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "t3oMFnwBEPmjP8ocCRiq",
"_score" : null,
"fields" : {
"@timestamp" : [
"2099-05-07T16:24:32.000Z"
]
},
"sort" : [
4081854272000
]
},
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "sHr7FXwBEPmjP8ocrxhf",
"_score" : null,
"fields" : {
"@timestamp" : [
"2099-05-06T16:21:15.000Z"
]
},
"sort" : [
4081767675000
]
}
]
}
}
- 时间范围查询
指定timestamp的时间范围大小 >= <=
指定具体的查询出来的列为timestamp
不展示原始JSON Object
GET logs-my_app-default/_search
{
"query": {
"range": {
"@timestamp": {
"gte": "2099-05-05",
"lt": "2099-05-08"
}
}
},
"_source": false,
"fields": [
"@timestamp"
],
"sort": [
{
"@timestamp": "desc"
}
]
}
结果:
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 2,
"relation" : "eq"
},
"max_score" : null,
"hits" : [
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "t3oMFnwBEPmjP8ocCRiq",
"_score" : null,
"fields" : {
"@timestamp" : [
"2099-05-07T16:24:32.000Z"
]
},
"sort" : [
4081854272000
]
},
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "sHr7FXwBEPmjP8ocrxhf",
"_score" : null,
"fields" : {
"@timestamp" : [
"2099-05-06T16:21:15.000Z"
]
},
"sort" : [
4081767675000
]
}
]
}
}
也有一些表达式，可以生成和当前日期相关的日期，例如：
"query": {
"range": {
"@timestamp": {
"gte": "now-1d/d",
"lt": "now/d"
}
}
},
- 从非结构化值中mapping出字段
1.请求中加入下面的mapping
2.在查询的Field中加入指定的类型
GET logs-my_app-default/_search
{
"runtime_mappings": {
"source.ip": {
"type": "ip",
"script": """
String sourceip=grok('%{IPORHOST:sourceip} .*').extract(doc[ "event.original" ].value)?.sourceip;
if (sourceip != null) emit(sourceip);
"""
}
},
"query": {
"range": {
"@timestamp": {
"gte": "2099-05-05",
"lt": "2099-05-08"
}
}
},
"_source": false,
"fields": [
"@timestamp",
"source.ip"
],
"sort": [
{
"@timestamp": "desc"
}
]
}
展示部分的查询结果：
截取部分查询结果，里面已经多了source.ip，且该字段是从原始文档中的event.original里面抽取出来
{
"_index" : ".ds-logs-my_app-default-2021.09.24-000001",
"_type" : "_doc",
"_id" : "sHr7FXwBEPmjP8ocrxhf",
"_score" : null,
"fields" : {
"@timestamp" : [
"2099-05-06T16:21:15.000Z"
],
"source.ip" : [
"192.0.2.42"
]
},
"sort" : [
4081767675000
]
}
- 复杂条件查询组合
在定义mapping抽取source.ip和时间范围查询的基础上。通过多source.ip 和 时间范围一起做为过滤条件：
修改QUERY命令：
"query": {
"bool": {
"filter": [
{
"range": {
"@timestamp": {
"gte": "2099-05-05",
"lt": "2099-05-08"
}
}
},
{
"range": {
"source.ip": {
"gte": "192.0.2.0",
"lte": "192.0.2.240"
}
}
}
]
}
},
Bool查询的作用：
相当于 and， 对bool下面的多个条件要同时符合的数据才能够被筛选出来。
- 聚合操作
在aggs选中聚合的列，然后最终的结果会把聚合的结果放在JSON的尾部
GET logs-my_app-default/_search
{
"runtime_mappings": {
"source.ip": {
"type": "ip",
"script": """
String sourceip=grok('%{IPORHOST:sourceip} .*').extract(doc[ "event.original" ].value)?.sourceip;
if (sourceip != null) emit(sourceip);
"""
},
"http.response.body.bytes": {
"type": "long",
"script": """
String bytes=grok('%{COMMONAPACHELOG}').extract(doc[ "event.original" ].value)?.bytes;
if (bytes != null) emit(Integer.parseInt(bytes));
"""
}
},
"query": {
"bool": {
"filter": [
{
"range": {
"@timestamp": {
"gte": "2099-05-05",
"lt": "2099-05-08"
}
}
}
]
}
},
"aggs": {
"http.response.body.bytes": {
"avg": {
"field": "http.response.body.bytes"
}
}
},
"_source": false,
"fields": [
"@timestamp",
"source.ip",
"http.response.body.bytes"
],
"sort": [
{
"@timestamp": "desc"
}
]
}
结果：
{
...
"aggregations" : {
"average_response_size" : {
"value" : 12368.0
}
}
}
删除数据
DELETE _data_stream/logs-my_app-default
