工作随笔 - shell脚本安装鉴权版本zookeeper/kafka集群

记录日常工作中的zk/kafka安装

组件	版本	安装包名
openjdk	1.8.0_345
zookeeper	3.7.1	apache-zookeeper-3.7.1-bin.tar.gz
kafka	2.12-2.5.1	kafka_2.12-2.5.1.tgz
centos	7.6

文件夹列表

.
├── apache-zookeeper-3.7.1-bin.tar.gz
├── install_zk_kafka.sh
├── jaas
│   ├── zk_client.conf
│   └── zk_server.conf
├── jaas.conf
├── java.env
├── kafka_2.12-2.5.1.tgz
└── kafka_client.properties

jaas.conf

KafkaServer {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="admin"
    password="pwd4test"
    user_admin="pwd4test"
    user_kafka="pwd4test";
};

ZkClient {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    username="kafka"
    password="pwd4test";

java.env

SERVER_JVMFLAGS="-Djava.security.auth.login.config=/home/test/App/zookeeper/conf/jaas/zk_server.conf -Dzookeeper.allowSaslFailedClients=false -Dzookeeper.sessionRequireClientSASLAuth=true"
CLIENT_JVMFLAGS="${CLIENT_JVMFLAGS} -Djava.security.auth.login.config=/home/test/App/zookeeper/conf/jaas/zk_client.conf"

kafka_client.properties

security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="pwd4test";

jaas/zk_client.conf

Client {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    username="kafka"
    password="pwd4test";
};

jaas/zk_server.conf

QuorumServer {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    user_admin="pwd4test";
};

QuorumLearner {
    org.apache.zookeeper.server.auth.DigestLoginModule required
    username="admin"
    password="pwd4test";
};

Server{
    org.apache.zookeeper.server.auth.DigestLoginModule required
    user_kafka="pwd4test";
};

install_zk_kafka.sh

#!/bin/bash
# usage  : sh install_zk_kafka.sh zookeeper|kafka|clean

BASE_DIR=$(cd `dirname $0`; pwd)
ZK_PACKAGE="apache-zookeeper-3.7.1-bin.tar.gz"
ZK_SERVER=(192.168.1.1 92.168.1.2 92.168.1.3)
LOCAL_IP=$(ifconfig|grep inet|grep -v 100.100| sed -n '1p'|awk '{print $2}')
ZK_PORT="2181"
INSTALL_BASE_PATH="/home/test"
KAFKA_PACKAGE="kafka_2.12-2.5.1.tgz"
KAFKA_PORT="9092"
RUN_ID="test"

function log_error() {
    echo -e "\033[31m [ERROR] $@ \033[0m"
}

function log_info() {
    echo -e "\033[32m [INFO] $@ \033[0m"
}

function log_warn() {
    echo -e "\033[33m [WARN] $@ \033[0m"
}

function check_jdk() {
    java_version=$(java -version 2>&1|grep version|awk -F '"' '{print $2}')
    
    if [ "${java_version}"x == x ];then
        log_error "未找到JDK,请事先安装JDK"
        exit 1
    else
        log_info "Java版本为:java_version"
    fi
    
}

function prepare_zk() {
    log_info "创建zookeeper所需文件夹"
    mkdir -pv ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs/zookeeper ${INSTALL_BASE_PATH}/Data/zookeeper    
}

function install_zk() {
    log_info "解压安装包${ZK_PACKAGE}"
    tar -xzvf ${BASE_DIR}/${ZK_PACKAGE} -C  ${INSTALL_BASE_PATH}/App/ 2>&1 > /dev/null
    mv ${INSTALL_BASE_PATH}/App/apache-zookeeper-3.7.1-bin ${INSTALL_BASE_PATH}/App/zookeeper

    log_info "创建配置文件${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg"
cat >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg <<EOF
tickTime=2000
initLimit=10
syncLimit=5
maxClientCnxns=500
autopurge.snapRetainCount=16
autopurge.purgeInterval=168
admin.enableServer=false
4lw.commands.whitelist=*
quorum.auth.enableSasl=true
quorum.auth.learnerRequireSasl=true
quorum.auth.serverRequireSasl=true
quorum.auth.learner.saslLoginContext=QuorumLearner
quorum.auth.server.saslLoginContext=QuorumServer
quorum.cnxn.threads.size=6
authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
requireClientAuthScheme=sasl
jaasLoginRenew=3600000
EOF

    echo "dataDir=${INSTALL_BASE_PATH}/Data/zookeeper" >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg
    echo "dataLogDir=${INSTALL_BASE_PATH}/Logs/zookeeper"  >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg
    echo "clientPort=${ZK_PORT}" >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg

    # get the server id
    server_id=""

    # update the server ip in config file
    i=0
    while [ $i -lt ${#ZK_SERVER[@]} ]
    do
        num=$i
        let i++
        echo "server.$i=${ZK_SERVER[$num]}:2888:3888" >> ${INSTALL_BASE_PATH}/App/zookeeper/conf/zoo.cfg
        [ "${LOCAL_IP}" == "${ZK_SERVER[$num]}" ] && log_info "server ID is $i" && server_id=$i && echo "$server_id" >> ${INSTALL_BASE_PATH}/Data/zookeeper/myid
    done

    # 判断local_ip是否属于节点中的一个,否则中断
    if [[ x"${server_id}" == "x"  ]];then
            log_error "脚本获取的LOCAL_IP(${LOCAL_IP})不在列表${ZK_SERVER[@]}中,请手动设置LOCAL_IP变量"
        exit 1
    fi

    log_info "添加zookeeper到环境变量"
    echo "export ZK_HOME=${INSTALL_BASE_PATH}/App/zookeeper" >> /etc/profile
    echo "export PATH=\$ZK_HOME/bin:\$PATH" >> /etc/profile

    log_info "添加zookeeper鉴权文件"
    cp -r ${BASE_DIR}/jaas ${INSTALL_BASE_PATH}/App/zookeeper/conf/
    cp ${BASE_DIR}/java.env ${INSTALL_BASE_PATH}/App/zookeeper/conf/
    sed -ri "s#/home/finance#${INSTALL_BASE_PATH}#g" ${INSTALL_BASE_PATH}/App/zookeeper/conf/java.env
    chmod 755 ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs ${INSTALL_BASE_PATH}/Data
    chown -R ${RUN_ID}:${RUN_ID} ${INSTALL_BASE_PATH}/App/zookeeper ${INSTALL_BASE_PATH}/Logs/zookeeper ${INSTALL_BASE_PATH}/Data/zookeeper
}

function start_zk() {
    log_info "启动zookeeper..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh start"
    sleep 5

    
    log_info "##########################"
    log_info "### 请记录以下鉴权信息 ###"
    log_info "zookeeper鉴权: kafka / pwd4test"
    log_info "##########################"
    log_warn "[注意]: 请在[第三个节点]按照完成之后,执行命令查看启动状态:source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh status"

}

function stop_zk() {
    log_info "停止zookeeper..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh stop"
    sleep 5
}

function status_zk() {
    log_info "zookeeper的status如下:"
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh status"
}

function prepare_kafka() {
    log_info "创建kafka所需文件夹"
    mkdir -pv ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs/kafka ${INSTALL_BASE_PATH}/Data/kafka    
}

function check_zk() {
    zk_status=$(source /etc/profile;${INSTALL_BASE_PATH}/App/zookeeper/bin/zkServer.sh status)
    zk_mode=$(echo $zk_status | grep Mode)

    if [ "${zk_mode}"x == x ];then
        log_error "未找到zookeeper,请事先安装zookeeper"
        exit 1
    else
        log_info "发现zookeeper运行,继续执行安装kafka"
    fi

}    

function install_kafka() {
    log_info "解压安装包${KAFKA_PACKAGE}"
    tar -xzvf ${BASE_DIR}/${KAFKA_PACKAGE} -C  ${INSTALL_BASE_PATH}/App/ 2>&1 > /dev/null
    #mv ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1 ${INSTALL_BASE_PATH}/App/kafka

    log_info "创建配置文件${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties"

cat >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties <<EOF
delete.topic.enable=true
num.network.threads=5
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
num.partitions=3
default.replication.factor=3
min.insync.replicas=2
num.recovery.threads.per.data.dir=6
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=2
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connection.timeout.ms=60000
group.initial.rebalance.delay.ms=10000
auto.create.topics.enable=true
unclean.leader.election.enable=false
security.inter.broker.protocol=SASL_PLAINTEXT
sasl.mechanism.inter.broker.protocol=PLAIN
sasl.enabled.mechanisms=PLAIN
allow.everyone.if.no.acl.found=true
EOF

    # get the server id
    server_id=""

    # update the server ip in config file
    i=0
    while [ $i -lt ${#ZK_SERVER[@]} ]
    do
        num=$i
        let i++
        [ "${LOCAL_IP}" == "${ZK_SERVER[$num]}" ] && echo "server ID is $i" && server_id=$i && echo "broker.id=${server_id}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    done

    # 判断local_ip是否属于节点中的一个,否则中断
    if [[ x"${server_id}" == "x"  ]];then
            log_error "脚本获取的LOCAL_IP(${LOCAL_IP})不在列表${ZK_SERVER[@]}中,请手动设置LOCAL_IP变量"
        exit 1
    fi

    echo "listeners=SASL_PLAINTEXT://${LOCAL_IP}:${KAFKA_PORT}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    echo "advertised.listeners=SASL_PLAINTEXT://${LOCAL_IP}:${KAFKA_PORT}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    echo "log.dirs=${INSTALL_BASE_PATH}/Data/kafka" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties
    echo "zookeeper.connect=${ZK_SERVER[0]}:${ZK_PORT},${ZK_SERVER[1]}:${ZK_PORT},${ZK_SERVER[2]}:${ZK_PORT}" >> ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties

    log_info "创建kafka鉴权文件"
    cp  ${BASE_DIR}/jaas.conf  ${BASE_DIR}/kafka_client.properties ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/

    log_info "更改kafka日志路径为: ${INSTALL_BASE_PATH}/Logs/kafka"
    sed -ri "s#LOG_DIR=\".*#LOG_DIR=\"${INSTALL_BASE_PATH}\/Logs\/kafka\"#g" ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-run-class.sh

    log_info "修改配置,开启kafka鉴权"
    sed -ri "s#KAFKA_OPTS=\"\"#KAFKA_OPTS=\"$KAFKA_OPTS -Djava.security.auth.login.config=${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/jaas.conf\"#g" ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-run-class.sh
    sed -ri "s#KAFKA_JVM_PERFORMANCE_OPTS=\"-server#KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -XX:MaxInlineLevel=15 -Djava.awt.headless=true -Dzookeeper.sasl.client=true -Dzookeeper.sasl.clientconfig=ZkClient -Dzookeeper.sasl.client.username=kafka -Djava.security.auth.login.config=${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/jaas.conf#g" ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-run-class.sh


    chmod 755 ${INSTALL_BASE_PATH}/App ${INSTALL_BASE_PATH}/Logs ${INSTALL_BASE_PATH}/Data
    chown -R ${RUN_ID}:${RUN_ID} ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1 ${INSTALL_BASE_PATH}/Logs/kafka ${INSTALL_BASE_PATH}/Data/kafka
}

function start_kafka() {
    log_info "启动kafka..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-server-start.sh -daemon ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/config/server.properties"
    sleep 5

    log_info "##########################"
    log_info "### 请记录以下鉴权信息 ###"
    log_info "kafka鉴权: admin / pwd4test"
    log_info "##########################"

    log_info "待三个节点启动完成,自行创建topic..."
}

function stop_kafka() {
    log_info "停止kafka..."
    su - ${RUN_ID} -c "source /etc/profile;${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1/bin/kafka-server-stop.sh"
    sleep 5
}

function clean() {
    log_warn "清理zookeeper..."
    rm -rf ${INSTALL_BASE_PATH}/App/zookeeper ${INSTALL_BASE_PATH}/Logs/zookeeper ${INSTALL_BASE_PATH}/Data/zookeeper
    log_warn "清理kafka..."
    rm -rf ${INSTALL_BASE_PATH}/App/kafka_2.12-2.5.1 ${INSTALL_BASE_PATH}/Logs/kafka ${INSTALL_BASE_PATH}/Data/kafka
    log_warn "清理完成..."
}

case "$1" in
  zookeeper)
        check_jdk
        prepare_zk
        install_zk
        start_zk
        #status_zk
        ;;
  kafka)
        check_jdk
        check_zk
        prepare_kafka
        install_kafka
        start_kafka
        ;;
  clean)
        stop_kafka
        stop_zk
        clean
        ;;        
  *)
        echo $"Usage: $0 {zookeeper|kafka|clean}"
        exit 2
esac

exit 0

使用方法

# 修改脚本中IP列表，确保三台服务器IP列表顺序一致
ZK_SERVER=(192.168.1.1 92.168.1.2 92.168.1.3)
# 三台服务器依次安装zookeeper
sh install_zk_kafka.sh zookeeper
# 待三台zookeeper安装完成，再依次安装kafka
sh install_zk_kafka.sh kafka

# [慎重执行]清理操作，清理所有zookeeper/kafka安装目录,以便重装
sh install_zk_kafka.sh clean

# zookeeper鉴权: kafka / pwd4test
# kafka鉴权: admin / pwd4test以下为纯手动安装配置方式

最后编辑于：2022.11.18 15:32:03

人面猴
序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
沈念sama阅读 204,445评论 6赞 478
死咒
序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
沈念sama阅读 85,889评论 2赞 381
救了他两次的神仙让他今天三更去死
文/潘晓璐我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
开封第一讲书人阅读 151,047评论 0赞 337
道士缉凶录：失踪的卖姜人
文/不坏的土叔我叫张陵，是天一观的道长。经常有香客问我，道长，这世上最难降的妖魔是什么？我笑而不...
开封第一讲书人阅读 54,760评论 1赞 276
港岛之恋（遗憾婚礼）
正文为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
茶点故事阅读 63,745评论 5赞 367
恶毒庶女顶嫁案：这布局不是一般人想出来的
文/花漫我一把揭开白布。她就那样静静地躺着，像睡着了一般。火红的嫁衣衬着肌肤如雪。梳的纹丝不乱的头发上，一...
开封第一讲书人阅读 48,638评论 1赞 281
城市分裂传说
那天，我揣着相机与录音，去河边找鬼。笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。我是一名探鬼主播，决...
沈念sama阅读 38,011评论 3赞 398
双鸳鸯连环套：你想象不到人心有多黑
文/苍兰香墨我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
开封第一讲书人阅读 36,669评论 0赞 258
万荣杀人案实录
序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
沈念sama阅读 40,923评论 1赞 299
护林员之死
正文独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋以下内容为张勋视角年9月15日...
茶点故事阅读 35,655评论 2赞 321
白月光启示录
正文我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
茶点故事阅读 37,740评论 1赞 330
活死人
序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
沈念sama阅读 33,406评论 4赞 320
日本核电站爆炸内幕
正文年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
茶点故事阅读 38,995评论 3赞 307
男人毒药：我在死后第九天来索命
文/蒙蒙一、第九天我趴在偏房一处隐蔽的房顶上张望。院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
开封第一讲书人阅读 29,961评论 0赞 19
一桩弑父案，背后竟有这般阴谋
文/苍兰香墨我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。一阵脚步声响...
开封第一讲书人阅读 31,197评论 1赞 260
情欲美人皮
我被黑心中介骗来泰国打工，没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。一个月前我还...
沈念sama阅读 45,023评论 2赞 350
代替公主和亲
正文我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
茶点故事阅读 42,483评论 2赞 342

工作随笔 - shell脚本安装鉴权版本zookeeper/kafka集群

文件夹列表

jaas.conf

java.env

kafka_client.properties

jaas/zk_client.conf

jaas/zk_server.conf

install_zk_kafka.sh

使用方法

推荐阅读更多精彩内容