51工具盒子

依楼听风雨
笑看云卷云舒,淡观潮起潮落

安装elk

# 安装 elk {#安装-elk}

本文讲述 elk(elastic、logstash,kibana)的安装步骤。logstash 负责从众多数据源(mysql、kafka 等)收集数据, 然后存储到 elastic, 最后通过 kibana 将数据可视化到界面上。elk 常用于日志收集和展示等。

# 1. 约定 {#_1-约定}

操作系统:Ubuntu 16.04.3 LTS

# 2. 准备资源 {#_2-准备资源}

jdk-8u144-linux-x64.tar.gz
elasticsearch-6.1.1.tar.gz
logstash-6.1.1.tar.gz
kibana-6.1.1-linux-x86_64.tar.gz
mysql-connector-java-5.1.41.jar

# 3. 安装 {#_3-安装}

# 3.1 安装前准备操作 {#_3-1-安装前准备操作}

注意

所有安装操作都在 docker 内执行

  • 创建容器环境 docker run -itd --privileged --name elk-ubuntu -p 9100:9100 -p 9200:9200 -p 9300:9300 -p 5601:5601 ubuntu:16.04
    docker cp /data/programInstaller/elasticsearch-6.1.1.tar.gz 28:/root/
    docker cp /data/programInstaller/jdk-8u144-linux-x64.tar.gz 28:/root/
    docker cp /data/programInstaller/kibana-6.1.1-linux-x86_64.tar.gz 28:/root/
    docker cp /data/programInstaller/logstash-6.1.1.tar.gz 28:/root/
    docker cp /data/programInstaller/mysql-connector-java/5.1.41/mysql-connector-java-5.1.41.jar 28:/root/
    docker exec -it 28 bash
    apt-get update
    apt-get install vim curl net-tools -y

  • 优化配置 vi /etc/security/limits.conf

    • soft nofile 65536
    • hard nofile 131072
    • soft nproc 2048
    • hard nproc 4096

vi /etc/security/limits.d/90-nproc.conf

-          soft    nproc     4096
  root soft nproc unlimited

vi /etc/sysctl.conf

vm.max_map_count=262144

sysctl -p

cd /root
tar -zxvf /root/jdk-8u144-linux-x64.tar.gz -C /usr/local/
vi /etc/profile.d/jdk.sh

#java
export JAVA_HOME=/usr/local/jdk1.8.0_144
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

source /etc/profile

tar -zxvf /root/elasticsearch-6.1.1.tar.gz -C /usr/local/
tar -zxvf /root/kibana-6.1.1-linux-x86_64.tar.gz -C /usr/local/
tar -zxvf /root/logstash-6.1.1.tar.gz -C /usr/local/
cp /root/mysql-connector-java-5.1.41.jar /usr/local/logstash-6.1.1/
groupadd elkusergroup
useradd elkuser -g elkusergroup -m
chown -R elkuser:elkusergroup /usr/local/elasticsearch-6.1.1/
chown -R elkuser:elkusergroup /usr/local/kibana-6.1.1-linux-x86_64/
chown -R elkuser:elkusergroup /usr/local/logstash-6.1.1/
su elkuser

# 3.2 安装 elasticsearch {#_3-2-安装-elasticsearch}

/usr/local/elasticsearch-6.1.1/bin/elasticsearch-plugin install x-pack
/usr/local/elasticsearch-6.1.1/bin/elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v6.1.1/elasticsearch-analysis-ik-6.1.1.zip
vi /usr/local/elasticsearch-6.1.1/config/elasticsearch.yml

network.host: 0.0.0.0
cluster.name: my-elastic-cluster1
node.name: elastic-node-1 #如果禁用安全组件,那么就没有了用户管理功能;如果要集成 logstash,那么暂时先做如下配置才能对接成功,否则可能需要做 ssl 相关配置才行 #如果没有安装 x-pack,则不需要如下配置
xpack.security.enabled: false

vi /usr/local/elasticsearch-6.1.1/config/jvm.options

-Xms3g
-Xmx3g

/usr/local/elasticsearch-6.1.1/bin/elasticsearch --verbose -d

设置 elastic,kibana,logstash_system 的密码[约定都设置为 123456].用户名分别为 elastic,kibana,logstash_system #如果没有安装 x-pack,则不需要如下设置密码操作
/usr/local/elasticsearch-6.1.1/bin/x-pack/setup-passwords interactive

# 3.3 安装 logstash {#_3-3-安装-logstash}

/usr/local/logstash-6.1.1/bin/logstash-plugin install x-pack
注意

需要确保源数据库的表中含有 update_time 字段

vi /usr/local/logstash-6.1.1/config/logstash.conf

input {
stdin {
}
jdbc {
jdbc_connection_string => "jdbc:mysql://172.16.38.73:3306/beauty2"
jdbc_user => "root"
jdbc_password => "123456"
jdbc_driver_library => "/usr/local/logstash-6.1.1/mysql-connector-java-5.1.41.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"

statement_filepath => "/usr/local/logstash-6.1.1/jdbc.sql"
use_column_value => "true"
tracking_column => "update_time"
schedule => "\* \* \* \* \*"
}
}

output {
elasticsearch {
hosts => ["http://localhost:9200"]

       index => "beauty2"
       document_type => "gallery"
       document_id => "%{id}"

       template => "/usr/local/logstash-6.1.1/template/logstash.json"
       template_overwrite => true
    }
    stdout{
       codec=>json_lines
    }

}

vi /usr/local/logstash-6.1.1/jdbc.sql

SELECT \* FROM sys_gallery where update_time > :sql_last_value

mkdir /usr/local/logstash-6.1.1/template
vi /usr/local/logstash-6.1.1/template/logstash.json

{
"template": "_",
"version": 60001,
"settings": {
"index": {
"refresh*interval": "5s"
}
},
"mappings": {
"\_default*": {
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart"
}
}
},
{
"string_fields": {
"match": "_",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"norms": false,
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "keyword"
},
"geoip": {
"dynamic": true,
"properties": {
"ip": {
"type": "ip"
},
"location": {
"type": "geo_point"
},
"latitude": {
"type": "half_float"
},
"longitude": {
"type": "half_float"
}
}
}
}
}
},
"aliases": {}
}

/usr/local/logstash-6.1.1/bin/logstash -f /usr/local/logstash-6.1.1/config/logstash.conf

# 3.4 安装 kibana {#_3-4-安装-kibana}

/usr/local/kibana-6.1.1-linux-x86_64/bin/kibana-plugin install x-pack
vi /usr/local/kibana-6.1.1-linux-x86_64/config/kibana.yml

server.host: "0.0.0.0"
elasticsearch.url: "http://localhost:9200"

#如果没有安装 x-pack,则不需要如下 2 行配置
elasticsearch.username: "elastic"
elasticsearch.password: "123456"

nohup /usr/local/kibana-6.1.1-linux-x86_64/bin/kibana >/usr/local/kibana-6.1.1-linux-x86_64/log 2>&1 &

# 3.5 安装 elasticsearch-head {#_3-5-安装-elasticsearch-head}

可以安装独立的 head 服务[默认端口 9100],也可以改用 chrome 浏览器的 elasticsearch-head 插件

# 4. 安装验证 {#_4-安装验证}

注意

可在宿主机操作

# 4.1 验证 elastic {#_4-1-验证-elastic}

浏览器访问:http://localhost:9200/?pretty
提示

账号密码(安装步骤中已设置):elastic/123456

# 4.2 验证 kibana {#_4-2-验证-kibana}

浏览器访问:http://localhost:5601

账号密码(安装步骤中已设置):kibana/123456

# 5. 备注 {#_5-备注}

# 5.1 配置自定义词典 {#_5-1-配置自定义词典}

# 5.1.1 将自定义词典文件放到 web 服务 {#_5-1-1-将自定义词典文件放到-web-服务}

例如访问路径http://172.16.38.73:8081/meitu_ik_custom.dic
文件格式如下(要求 utf8 编码)

美臀
爆乳

# 5.1.2 配置 elasticsearch {#_5-1-2-配置-elasticsearch}

vi /usr/local/elasticsearch-6.1.1/config/analysis-ik/IKAnalyzer.cfg.xml

<!-- 指定远程扩展字典服务 -->
<entry key="remote_ext_dict">http://172.16.38.73:8081/meitu_ik_custom.dic</entry>

# 5.1.3 重启 elasticsearch {#_5-1-3-重启-elasticsearch}

注意

若想对历史文档应用自定义词典,则需要重新导入历史文档

赞(0)
未经允许不得转载:工具盒子 » 安装elk