脚本部署（CM=5.10.0）
创建工作目录workspace（自己定义）
上传以下包到工作目录
mysql-connector-java-5.1.40.jar
cloudera-manager-centos7-cm5.10.0_x86_64.tar.gz
CDH-5.10.0-1.cdh5.10.0.p0.41-el7.manifest.json
CDH-5.10.0-1.cdh5.10.0.p0.41-el7.parcel
CDH-5.10.0-1.cdh5.10.0.p0.41-el7.parcel.sha
jdk-8u60-linux-x64.rpm
ntp.conf
编辑hosts和workers，格式都是“ip hostname”
hosts是用来替换/etc/hosts的
workers是新装或增加时的机器列表
解压tar包
tar -xzf cloudera-manager-centos7-cm5.10.0_x86_64.tar.gz
修改/opt/cm-5.10.0/etc/cloudera-scm-agent/config.ini中server_host的值
运行distribute.sh进行环境准备
#!/bin/bash
CURRENT_DIR=$(cd $(dirname $0);pwd)
#cm版本
CM=cm-5.10.0
mv mysql-connector-java-5.1.40.jar cm-5.10.0/share/cmf/lib/
ln -s $CM/share/cmf/lib/mysql-connector-java-5.1.40.jar $CM/share/cmf/lib/mysql-connector-java.jar
#生成秘钥ssh-keygen
while read pair
do
host=${pair#* }
ip=${pair% *}
echo "distribute to$ip/$host"
#分发密钥
ssh-copy-id $ip
#替换hosts
scp $CURRENT_DIR/hosts $ip:/etc/hosts
#安装vim
ssh -n $ip yum install -y vim
#修改vim注释颜色为绿色
ssh -n $ip "echo " highlight Comment ctermfg=green guifg=green" >> /etc/vimrc"
#修改主机名
ssh -n $ip hostname $host
ssh -n $ip "echo "$hostname" > /etc/sysconfig/network"
ssh -n $ip "echo "NOZEROCONF=yes" >> /etc/sysconfig/network"
#安装lsof
ssh -n $ip yum install -y lsof
#安装ntp
ssh -n $ip yum install -y ntp
scp $CURRENT_DIR/ntp.conf $ip:/etc/ntp.conf
ssh -n $ip service ntpd restart
#安装psmisc
ssh -n $ip yum install -y psmisc
#启动rpcbind
ssh -n $ip service rpcbind start
#修改swap
ssh -n $ip "echo "0" > /proc/sys/vm/swappiness"
ssh -n $ip "echo "vm.swappiness=0" >> /etc/sysctl.conf"
#安装jdk
ssh -n $ip mkdir /usr/java/
scp $CURRENT_DIR/jdk-8u60-linux-x64.rpm $ip:/usr/java/
ssh -n $ip rpm -i /usr/java/jdk-8u60-linux-x64.rpm
ssh -n $ip ln -s /usr/java/jdk1.8.0_60 /usr/java/jdk
ssh -n $ip "echo "exportJAVA_HOME=/usr/java/jdk" >> /etc/profile"
ssh -n $ip "echo "exportPATH=\\\$JAVA_HOME/bin:\\\$PATH" >> /etc/profile"
ssh -n $ip "echo "export CLASSPATH=.:\\\$JAVA_HOME/lib/dt.jar:\\\$JAVA_HOME/lib/tools.jar" >> /etc/profile"
ssh -n $ip source/etc/profile
#创建cm安装主目录
ssh -n $ip mkdir /opt
#分发cm文件
scp -r $CURRENT_DIR/$CM/ $ip:/opt/
#创建cloudera-scm-server用户目录
ssh -n $ip useradd --system --home=/opt/$CM/run/cloudera-scm-server --no-create-home --shell=/bin/false--comment \"Cloudera SCM User\" cloudera-scm
done < $CURRENT_DIR/hosts
运行start-server.sh启动Server
#!/bin/bash
CURRENT_DIR=$(cd $(dirname $0);pwd)
CM=cm-5.10.0
user=root
password=123456
host=127.0.0.1
#创建日志目录
mkdir /var/log/cloudera-scm-server
chown cloudera-scm:cloudera-scm /var/log/cloudera-scm-server
#创建parcel仓库目录
mkdir -p /opt/cloudera/parcel-repo
chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
#创建parcel目录
mkdir -p /opt/cloudera/parcelschown cloudera-scm:cloudera-scm /opt/cloudera/parcels
#复制parcel到parcel仓库目录
cp $CURRENT_DIR/*.manifest.json /opt/cloudera/parcel-repo
cp $CURRENT_DIR/*.parcel /opt/cloudera/parcel-repo
cp $CURRENT_DIR/*.parcel.sha /opt/cloudera/parcel-repo
#创建元数据库
echo "create database cloudera_manager;create database reports_manager;create database oozie;create database hive;create database hue;"| mysql -u$user -p$password -h$host
#启动server
/opt/$CM/share/cmf/schema/scm_prepare_database.sh mysql cloudera_manager $user $password
/opt/$CM/etc/init.d/cloudera-scm-server start
运行start-agents.sh启动Agents
#!/bin/bash
CURRENT_DIR=$(cd $(dirname $0);pwd)
CM=cm-5.10.0
while read pair
do ip=${pair#* }
echo "start agent on$ip"
#启动agent
ssh -n $ip /opt/$CM/etc/init.d/cloudera-scm-agent start
done < $CURRENT_DIR/workers
MySQL方面
以下都要安装
mysql-community-client-5.7.12-1.el7.x86_64
mysql-community-devel-5.7.12-1.el7.x86_64
mysql-community-libs-5.7.12-1.el7.x86_64
mysql-community-server-5.7.12-1.el7.x86_64
mysql-community-common-5.7.12-1.el7.x86_64
mysql驱动问题可能导致hive canary的测试过不去
Hue
可能缺少以下各种mysql包
libexslt.so.0.8(.17)
libmysqlclient.so.18(.1.0)
libxslt.so.1(1.28)
hive sasl问题
yum install cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi
系统方面
缺少pstree解决办法：yum install psmisc -y
nfs启动失败：service rpcbind start
HDFS权限问题
将dfs.permission设为false
zookeeper connection reset by peer
zookeeper最大连接数问题
yarn模式问题
worker节点jar包缺失
hive mysql驱动问题
在/opt/cloudera/parcels/CDH/lib/hive/lib下建一个软链接
oozie mysql驱动问题
/var/lib/oozie/mysql-connector-java.jar
yarn日志级别
yarn容器大小
hdfs权限
zookeeper最大连接数设置
oozie日志级别
hue时区
