跳到主要内容

后端部署

2.1 准备工作​

除kibana外，都需在jdk11支持。

# tar zxvf jdk-11.0.6_linux-x64_bin.tar.gz
# mv jdk-11.0.6 /home/data/elk/jdk


# cat /etc/profile
#Java JDK/JRE config
export JAVA_HOME=/home/data/elk/jdk
export JDK_HOME=$JAVA_HOME
export JRE_HOME=$JAVA_HOME/jre
export PATH=.:$JDK_HOME/bin:$JRE_HOME/bin:$PATH
export CLASSPATH=.:$JDK_HOME/lib:$JDK_HOME/lib/dt.jar:$JDK_HOME/lib/tools.jar:$JRE_HOME/lib:$JRE_HOME/lib/javaws.jar:$JRE_HOME/lib/rt.jar:$JRE_HOME/lib/ext

# source /etc/profile
# java -version
java version "11.0.6" 2020-01-14 LTS
Java(TM) SE Runtime Environment 18.9 (build 11.0.6+8-LTS)
Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.6+8-LTS, mixed mode)

2.2 LB部署​

需部署2台

设备要求： 计算能力、内存、网络

2.2.1 部署​

安装

# yum -y install keepalived ipvsadm
# chkconfig --add keepalived
# chkconfig keepalived on

配置（DR方式） /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   router_id HA_181 不同节点配置不同
   vrrp_skip_check_adv_addr
   #vrrp_strict    注解该项
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER
    interface eth4
    virtual_router_id 180
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        198.76.20.180/24 dev eth5
        172.76.20.180/24 dev eth4
        172.76.20.119/24 dev eth4
    }

}

#es
virtual_server 172.76.20.119 9200 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 20
    protocol TCP

    real_server 172.76.20.124 9200 {
        weight 1
        TCP_CHECK {
           connect_port 9200
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.125 9200 {
        weight 1
        TCP_CHECK {
           connect_port 9200
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.147 9200 {
        weight 1
        TCP_CHECK {
           connect_port 9200
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }
}

#syslog-linux
virtual_server 172.76.20.119 7514 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 20
    protocol TCP

    real_server 172.76.20.124 7514 {
        weight 1
        TCP_CHECK {
           connect_port 7514
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.125 7514 {
        weight 1
        TCP_CHECK {
           connect_port 7514
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.147 7514 {
        weight 1
        TCP_CHECK {
           connect_port 7514
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

}

#syslog-net
virtual_server 172.76.20.119 7515 {
    delay_loop 20
    lb_algo rr
    lb_kind DR
    persistence_timeout 5
    protocol UDP

    real_server 172.76.20.124 7515 {
        weight 1
        TCP_CHECK {
           connect_port 7515
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.125 7515 {
        weight 1
        TCP_CHECK {
           connect_port 7515
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.147 7515 {
        weight 1
        TCP_CHECK {
           connect_port 7515
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

}

#x-beat
virtual_server 172.76.20.119 5044 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 5
    protocol TCP

    real_server 172.76.20.124 5044 {
        weight 1
        TCP_CHECK {
           connect_port 5044
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.125 5044 {
        weight 1
        TCP_CHECK {
           connect_port 5044
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.147 5044 {
        weight 1
        TCP_CHECK {
           connect_port 5044
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }


}

#test
virtual_server 172.76.20.119 5068 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 20
    protocol TCP

    real_server 172.76.20.124 5068 {
        weight 1
        TCP_CHECK {
           connect_port 5068
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.125 5068 {
        weight 1
        TCP_CHECK {
           connect_port 5068
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.147 5068 {
        weight 1
        TCP_CHECK {
           connect_port 5068
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

}

启动

# /etc/init.d/keepalived start

2.2.2 测试​

LB列表查看

# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  172.76.20.119:5044 rr persistent 5
TCP  172.76.20.119:5068 rr persistent 20
TCP  172.76.20.119:7514 rr persistent 20
TCP  172.76.20.119:9200 rr persistent 20
UDP  172.76.20.119:7515 rr persistent 5

VIP查看、掉电主节点，再查看

# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:7f:81:75 brd ff:ff:ff:ff:ff:ff
3: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:7f:81:76 brd ff:ff:ff:ff:ff:ff
4: eth2: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:7f:81:77 brd ff:ff:ff:ff:ff:ff
5: eth3: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:7f:81:78 brd ff:ff:ff:ff:ff:ff
6: eth4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
    link/ether 00:10:f3:76:53:c9 brd ff:ff:ff:ff:ff:ff
    inet 172.76.20.181/24 brd 172.76.20.255 scope global eth4
    inet 172.76.20.180/24 scope global secondary eth4
    inet 172.76.20.119/24 scope global secondary eth4
    inet6 fe80::210:f3ff:fe76:53c9/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
7: eth5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
    link/ether 00:10:f3:76:53:ca brd ff:ff:ff:ff:ff:ff
    inet 198.76.20.181/24 brd 198.76.20.255 scope global eth5
    inet 198.76.20.180/24 scope global secondary eth5
    inet6 fe80::210:f3ff:fe76:53ca/64 scope link 
       valid_lft forever preferred_lft forever
8: eth6: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:76:53:cb brd ff:ff:ff:ff:ff:ff
9: eth7: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:76:53:cc brd ff:ff:ff:ff:ff:ff
10: eth8: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:74:2a:a0 brd ff:ff:ff:ff:ff:ff
11: eth9: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
    link/ether 00:10:f3:74:2a:a1 brd ff:ff:ff:ff:ff:ff

2.3 Elasticsearch​

需部署3台.

设备要求：
计算能力、内存、硬盘

2.3.1 部署​

准备

修改/etc/security/limits.conf文件，增加配置，用户退出后重新登录生效

*               soft    nofile          65536
*               hard    nofile          65536
*               soft    nproc           4096
*               hard    nproc           4096

修改/etc/security/limits.d/90-nproc.conf，增加配置，用户退出后重新登录生效

*               soft    nproc           4096
*               hard    nproc           4096

修改vm.max_map_count

echo "vm.max_map_count=262144" >> /etc/sysctl.conf
sysctl -p

安装

# tar zxvf elasticsearch-7.6.1-linux-x86_64.tar.gz
# mv elasticsearch-7.6.1 /home/data/elk/elasticsearch
# chattr -i /etc/passwd /etc/shadow /etc/group /etc/gshadow
# useradd es
# chown -R es:es  /home/data/elk/elasticsearch                #elasticsearch不能以root身份运行
# chattr +i /etc/passwd /etc/shadow /etc/group /etc/gshadow

# vi /home/data/elk/elasticsearch/config/elasticsearch.yml    #此处需再参照x-pack配置
cluster.name: yd-elk
node.name: node-X                     #此参数，不同节点配置不同。
#network.host: 0.0.0.0
network.bind_host: 0.0.0.0
network.publish_host: 172.76.20.124   #此参数，不同节点配置不同
http.port: 9200
transport.tcp.port: 9300
discovery.seed_hosts: ["172.76.20.124", "172.76.20.125", "172.76.20.147"]            #配置自动发现的节点。
cluster.initial_master_nodes: ["172.76.20.124", "172.76.20.125", "172.76.20.147"]    #配置哪些节点参与主节点选举。

bootstrap.system_call_filter: false

xpack.ml.enabled: false
xpack.security.enabled: false

http.cors.enabled: true
http.cors.allow-origin: "*"

node.master: true                  #是否为主节点，可以有多个主节点。
node.data: true                    #是否为数据节点，可以有多个数据节点。
#action.auto_create_index: true    #允许自动给文档建立索引值。

启动

su - es -c "source /etc/profile; /home/data/elk/elasticsearch/bin/elasticsearch > /dev/null 2>&1 &"

提示： 可将启动命令添加到/etc/rc.local

es实时日志

tail -f /home/data/elk/elasticsearch/logs/yd-elk.log

添加VIP

ip addr add 172.76.20.119/32 broadcast 172.76.20.119 dev lo

配置Non-ARP

echo "net.ipv4.conf.lo.arp_ignore = 1" >> /etc/sysctl.conf
echo "net.ipv4.conf.lo.arp_announce = 2" >> /etc/sysctl.conf
echo "net.ipv4.conf.all.arp_ignore = 1" >> /etc/sysctl.conf
echo "net.ipv4.conf.all.arp_announce = 2" >> /etc/sysctl.conf
sysctl -p
提示

若不添加上述内核参数，虽通讯正常，但在接入层交换机日志中会提示如下日志
Detected an IP address conflict xxxxx

2.3.2 修改索引默认分片配置​

# curl -H "Content-Type: application/json" -XPUT http://127.0.0.1:9200/_template/log -d' 
{
  "template": "*",
  "settings": {
    "number_of_shards": "2",
    "number_of_replicas": "1"
  }
}'

2.3.3 测试​

3台es都启动正常后，在路由可达的任意主机上做如下测试，采用vip访问.

写入测试数据

# curl -H "Content-Type: application/json" -XPOST http://172.76.20.119:9200/test/member -d' {
   "sex":"man",
   "name":"FuSheng Guo"
}'

查看索引列表

# curl http://172.76.20.119:9200/_cat/indices?v
health status index                              uuid                   pri rep docs.count docs.deleted store.size pri.store.size
green  open   test                               yZ_L-lGdRM2r1iZPqHusjQ   3   2          0            0        2kb           690b

查看es集群中节点情况

# curl http://172.76.20.119:9200/_cat/nodes?v
ip            heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
172.76.20.124           34          83   0    0.37    0.35     0.45 dim       -      node-1
172.76.20.125           27          88   0    0.00    0.06     0.13 dim       *      node-2
172.76.20.147           21          21   3    0.42    0.42     0.26 dim       -      node-3

查看es集群健康状态

# curl http://172.76.20.119:9200/_cat/health?v
epoch      timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1590481266 08:21:06  yd-elk  green           3         3    189  73    0    0        0             0                  -                100.0%

查看LB表

在LB服务器上查看如下：

#  ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  172.76.20.119:5044 rr persistent 5
TCP  172.76.20.119:5068 rr persistent 20
TCP  172.76.20.119:7514 rr persistent 20
TCP  172.76.20.119:9200 rr persistent 20
  -> 172.76.20.124:9200           Route   1      0          0         
  -> 172.76.20.125:9200           Route   1      0          0         
  -> 172.76.20.147:9200           Route   1      0          3         
UDP  172.76.20.119:7515 rr persistent 5

2.4 Logstash​

需部署3台，与es同机配置。

设备要求：
计算能力、内存

2.4.1 部署​

安装

# wget https://artifacts.elastic.co/downloads/logstash/logstash-7.6.1.tar.gz
# tar zxvf logstash-7.6.1.tar.gz
# mv logstash-7.6.1 /home/data/elk/logstash

配置,此步为重点配置

  • Pip管道文件
# vi /home/data/elk/logstash/config/pipelines.yml
- pipeline.id: syslog-linux
  path.config: "/home/data/elk/logstash/config/conf.d/syslog-linux.conf"
- pipeline.id: syslog-net
  path.config: "/home/data/elk/logstash/config/conf.d/syslog-net.conf"
- pipeline.id: beats
  path.config: "/home/data/elk/logstash/config/conf.d/beats.conf"
- pipeline.id: test-http
  path.config: "/home/data/elk/logstash/config/conf.d/test-http.conf"
  • 应用配置文件,请查看附件
/home/data/elk/logstash/config/conf.d/syslog-linux.conf
/home/data/elk/logstash/config/conf.d/syslog-net.conf
/home/data/elk/logstash/config/conf.d/beats.conf
/home/data/elk/logstash/config/conf.d/test-http.conf

启动

su - root -c "source /etc/profile;/home/data/elk/logstash/bin/logstash > /dev/null 2>&1 &"

实时日志查看

tail -f /home/data/elk/logstash/logs/logstash-plain.log

2.4.2 测试​

请参照“后端联调测试”

# netstat -an | grep LISTEN
tcp        0      0 0.0.0.0:36863               0.0.0.0:*                   LISTEN      
tcp        0      0 127.0.0.1:199               0.0.0.0:*                   LISTEN      
tcp        0      0 0.0.0.0:3306                0.0.0.0:*                   LISTEN      
tcp        0      0 0.0.0.0:29997               0.0.0.0:*                   LISTEN      
tcp        0      0 127.0.0.1:25                0.0.0.0:*                   LISTEN      
tcp        0      0 :::36863                    :::*                        LISTEN      
tcp        0      0 ::ffff:127.0.0.1:9600       :::*                        LISTEN      
tcp        0      0 :::5068                     :::*                        LISTEN      
tcp        0      0 :::29997                    :::*                        LISTEN      
tcp        0      0 :::9200                     :::*                        LISTEN      
tcp        0      0 :::5044                     :::*                        LISTEN      
tcp        0      0 :::9300                     :::*                        LISTEN      
tcp        0      0 ::1:25                      :::*                        LISTEN      
tcp        0      0 :::7514                     :::*                        LISTEN      
tcp        0      0 :::7515                     :::*                        LISTEN  

# netstat -an | grep udp
udp        0      0 0.0.0.0:161                 0.0.0.0:*                               
udp        0      0 0.0.0.0:10801               0.0.0.0:*                               
udp        0      0 0.0.0.0:7514                0.0.0.0:*                               
udp        0      0 0.0.0.0:7515                0.0.0.0:*

2.5 Kibana​

需要1台即可。

提示：
Kibana不需要jdk支持。

2.5.1 部署​

  • 安装
tar zxvf kibana-7.6.1-linux-x86_64.tar.gz
mv kibana-7.6.1-linux-x86_64 /usr/local/kibana
  • 配置
# vi /usr/local/kibana/config/kibana.yml 
elasticsearch.hosts: ["http://172.76.20.119:9200"] 
i18n.locale: zh-CN
  • 启动
# /usr/local/kibana/bin/kibana serve -c /usr/local/kibana/config/kibana.yml --allow-root --host 0.0.0.0  > /dev/null 2>&1 &

2.5.2 测试​

http://172.76.20.119:5601/

2.6 后端联调测试​

2.6.1 测试目标​

模拟数据采集全过程，采集端--->LB----->Logstash----->ES----Kibana

2.6.2 Logstash添加测试侦听端口​

# cat test-http.conf 
input {
  http {
    host => "0.0.0.0"
    port => 5068
    type => "http-input"
  }
}

filter{
}

output {

  elasticsearch {
    hosts => ["172.76.20.119:9200"]
    index => "test-by-gfs-%{+YYYY.MM.dd}"
    #document_type => "beat-test-log"
  }

}

2.6.3 LB添加测试端口流量均衡​

在LB上添加，如下：

#test
virtual_server 172.76.20.119 5068 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 5
    protocol TCP

    real_server 172.76.20.124 5068 {
        weight 1
        TCP_CHECK {
           connect_port 5068
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

    real_server 172.76.20.125 5068 {
        weight 1
        TCP_CHECK {
           connect_port 5068
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
}

    real_server 172.76.20.147 5068 {
        weight 1
        TCP_CHECK {
           connect_port 5068
           connect_timeout 3
           #nb_get_retry 3
           retry 3
           delay_before_retry 3
        }
    }

2.6.4 测试过程​

  • 采集模拟 在任一台路由可达的主机上输入如下命令
# curl -H "Content-Type: application/json" -XPOST http://172.76.20.119:5068/ -d'
{
  "vCPU": "16",
  "RAM": "32G"
}'

  • es写入查看

  • logstash实时日志查看

tail -f /home/data/elk/logstash/logs/logstash-plain.log
  • es实时日志
tail -f /home/data/elk/elasticsearch/logs/yd-elk.log

Alt text