ceph分布式存储配置

1 ceph后端存储配置实战

1.1 ceph配置前的准备

1.1.1 三台机器都要配置

[root@linux-node1 ~]# hostname linux-node1

[root@linux-node1 ~]# echo "linux-node1" >>/etc/hostname

[root@linux-node2 ~]# hostname linux-node2

[root@linux-node2 ~]# echo "linux-node2" >>/etc/hostname

[root@linux-node3 ~]# hostname linux-node3

[root@linux-node3 ~]# echo "linux-node2" >>/etc/hostname

[root@linux-node1 ~]# vi /etc/hosts

192.168.100.151 linux-node1 linux-node1.oldboyedu.com

192.168.100.152 linux-node2 linux-node2.oldboyedu.com

192.168.100.153 linux-node3

[root@linux-node1 ~]# cd /etc/

[root@linux-node1 etc]# scp hosts linux-node2:/etc/

[root@linux-node1 etc]# scp hosts linux-node3:/etc/

[root@linux-node1 ~]# systemctl stop firewalld.service

[root@linux-node1 ~]# setenforce 0

[root@linux-node1 ~]# id ceph

[root@linux-node2 ~]# id ceph

[root@linux-node3 ~]# id ceph

[root@linux-node1 ~]# useradd -d /home/ceph -m ceph

[root@linux-node1 ~]# echo "ceph ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/ceph

[root@linux-node1 ~]# passwd ceph

Changing password for user ceph.

New password:

BAD PASSWORD: The password is shorter than 8 characters

Retype new password:

passwd: all authentication tokens updated successfully.

[root@linux-node2 ~]# passwd ceph

Changing password for user ceph.

New password:

BAD PASSWORD: The password is shorter than 8 characters

Retype new password:

passwd: all authentication tokens updated successfully.

[root@linux-node3 ~]# passwd ceph

Changing password for user ceph.

New password:

BAD PASSWORD: The password is shorter than 8 characters

Retype new password:

passwd: all authentication tokens updated successfully.

[root@linux-node1 ~]# chmod 0440 /etc/sudoers.d/ceph

[root@linux-node1 ~]# visudo

#修改配置文件如下

... ...

Defaults:ceph !requiretty

[root@linux-node1 ~]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

[root@linux-node2 ~]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

[root@linux-node3 ~]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

[root@linux-node1 ~]# cd /etc/yum.repo.d/

[root@linux-node1 yum.repo.d]# vi ceph.repo

[ceph]

name=ceph

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/

gpgcheck=0

priority=1

[ceph-noarch]

name=cephnoarch

baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/

gpgcheck=0

priority=1

[ceph-source]

name=Ceph source packages

baseurl=https://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/

enabled=0

gpgcheck=1

type=rpm-md

gpgkey=http://mirrors.aliyun.com/ceph/keys/release.asc

priority=1

[root@linux-node1 ~]# yum clean all

[root@linux-node1 ~]# yum makecache

[root@linux-node1 ~]# yum repolist

1.1.2 在linux-node1配置SSH认证

[ceph@linux-node1 ~]$ ssh-keygen -t rsa

[ceph@linux-node1 ~]$ ssh-copy-id ceph@linux-node1

[ceph@linux-node1 ~]$ ssh ceph@linux-node1

[ceph@linux-node1 ~]$ ssh-copy-id ceph@linux-node2

[ceph@linux-node1 ~]$ ssh ceph@linux-node2

[ceph@linux-node1 ~]$ ssh-copy-id ceph@linux-node3

[ceph@linux-node1 ~]$ ssh ceph@linux-node3

[ceph@linux-node1 ~]$ sudo vim ~/.ssh/config

Host linux-node1

Hostname linux-node1

User ceph

Host linux-node2

Hostname linux-node2

User ceph

Host linux-node3

Hostname linux-node3

User ceph

[ceph@linux-node1 ~]$ sudo chmod 600 ~/.ssh/config

1.2 安装和配置ceph

[ceph@linux-node1 ~]$ sudo yum install yum-plugin-priorities

[ceph@linux-node1 ~]$ sudo yum install ceph-deploy

[ceph@linux-node1 ~]$ sudo mkdir my-cluster

[ceph@linux-node1 ~]$ sudo cd my-cluster

[ceph@linux-node1 ~]$ sudo chown -R ceph.ceph /home/ceph/my-cluster/

[ceph@linux-node1 my-cluster]$ ceph-deploy new linux-node1 linux-node2 linux-node3

[ceph@linux-node1 my-cluster]$ ceph-deploy install --no-adjust-repos linux-node1 linux-node2 linux-node3

[ceph@linux-node1 my-cluster]$ ceph-deploy mon create-initial

[ceph@linux-node1 my-cluster]$ ceph-deploy disk zap linux-node1 /dev/sdb

[ceph@linux-node1 my-cluster]$ ceph-deploy disk zap linux-node2 /dev/sdb

[ceph@linux-node1 my-cluster]$ ceph-deploy disk zap linux-node3 /dev/sdb

[ceph@linux-node1 my-cluster]$ sudo chmod +r ceph.client.admin.keyring

[ceph@linux-node1 my-cluster]$ sudo cp -a /home/ceph/my-cluster/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring

[ceph@linux-node1 my-cluster]$ vi ~/my-cluster/ceph.conf

[global]

fsid = 55b2d4e8-cb32-4edb-9cce-d32a64294503

mon_initial_members = linux-node1, linux-node2, linux-node3

mon_host = 192.168.100.151,192.168.100.152,192.168.100.153

auth_cluster_required = cephx

auth_service_required = cephx

auth_client_required = cephx

osd pool default size = 2

mon clock drift allowed = 2

mon clock drift warn backoff = 30

[mon]

mon allow pool delete = true

[ceph@linux-node1 my-cluster]$ ceph-deploy --overwrite-conf config push linux-node{1..3}

[ceph@linux-node1 my-cluster]$ sudo systemctl restart ceph-mon.target

[ceph@linux-node1 my-cluster]$ sudo systemctl stauts ceph-mon.target

[ceph@linux-node1 my-cluster]$ ceph-deploy osd create linux-node1 --data /dev/sdb

[ceph@linux-node1 my-cluster]$ ceph-deploy osd create linux-node2 --data /dev/sdb

[ceph@linux-node1 my-cluster]$ ceph-deploy osd create linux-node3 --data /dev/sdb

[ceph@linux-node1 my-cluster]$ ceph-deploy mgr create linux-node1 linux-node2 linux-node3

[ceph@linux-node1 my-cluster]$ ceph -s

[ceph@linux-node1 my-cluster]$ vi ceph.conf

[global]

fsid = 55b2d4e8-cb32-4edb-9cce-d32a64294503

mon_initial_members = linux-node1, linux-node2, linux-node3

mon_host = 192.168.100.151,192.168.100.152,192.168.100.153

auth_cluster_required = cephx

auth_service_required = cephx

auth_client_required = cephx

osd pool default size = 2

mon clock drift allowed = 2

mon clock drift warn backoff = 30

[mon]

mon allow pool delete = true

[ceph@linux-node1 my-cluster]$ ceph-deploy --overwrite-conf config push linux-node{1..3}

[ceph@linux-node1 my-cluster]$ sudo systemctl restart ceph-mon.target

[ceph@linux-node1 my-cluster]$ sudo systemctl stauts ceph-mon.target

1.3 创建文件系统

1.3.1 创建Ceph MDS角色

[ceph@linux-node1 my-cluster]$ sudo modprobe rbd

[ceph@linux-node1 my-cluster]$ sudo lsmod | grep rbd

[ceph@linux-node1 my-cluster]$ sudo modprobe ceph

[ceph@linux-node1 my-cluster]$ sudo lsmod | grep ceph

1.3.2 创建Ceph MDS角色

[ceph@linux-node1 my-cluster]$ ceph-deploy --overwrite-conf mds create linux-node1

[ceph@linux-node1 my-cluster]$ netstat -tnlp | grep mds

1.3.3 手动创建data和metadata两个池

[ceph@linux-node1 my-cluster]$ ceph osd pool create cephfs_data 128

[ceph@linux-node1 my-cluster]$ ceph osd pool create cephfs_metadata 128

[ceph@linux-node1 my-cluster]$ ceph osd pool stats cephfs_data

[ceph@linux-node1 my-cluster]$ ceph osd pool stats cephfs_metadata

[ceph@linux-node1 my-cluster]$ ceph fs new cephfs cephfs_metadata cephfs_data

[ceph@linux-node1 my-cluster]$ ceph fs ls

[ceph@linux-node1 my-cluster]$ ceph mds stat

[ceph@linux-node1 ~]$ mysql my-cluster/

[ceph@linux-node1 my-cluster]$ vi ceph.conf

[global]

fsid = 55b2d4e8-cb32-4edb-9cce-d32a64294503

mon_initial_members = linux-node1, linux-node2, linux-node3

mon_host = 192.168.100.151,192.168.100.152,192.168.100.153

auth_cluster_required = cephx

auth_service_required = cephx

auth_client_required = cephx

osd pool default size = 2

mon clock drift allowed = 2

mon clock drift warn backoff = 30

[mon]

mon allow pool delete = true

[ceph@linux-node1 my-cluster]$ sudo systemctl restart ceph-mon.target

[ceph@linux-node1 my-cluster]$ sudo systemctl stauts ceph-mon.target

[ceph@linux-node1 my-cluster]$ ceph -w

cluster:

id: b86c386a-141f-4330-8267-2b15584bb915

health: HEALTH_OK

services:

mon: 3 daemons, quorum linux-node1,linux-node2,linux-node3

mgr: linux-node1(active), standbys: linux-node2, linux-node3

mds: cephfs-1/1/1 up {0=linux-node1=up:active}

osd: 3 osds: 3 up, 3 in

data:

pools: 2 pools, 256 pgs

objects: 21 objects, 2246 bytes

usage: 3080 MB used, 58347 MB / 61428 MB avail

pgs: 256 active+clean

2018-08-15 18:27:10.171244 mon.linux-node1 [INF] overall HEALTH_OK

[ceph@linux-node1 my-cluster]$ ceph -s

cluster:

id: b86c386a-141f-4330-8267-2b15584bb915

health: HEALTH_OK

services:

mon: 3 daemons, quorum linux-node1,linux-node2,linux-node3

mgr: linux-node1(active), standbys: linux-node2, linux-node3

mds: cephfs-1/1/1 up {0=linux-node1=up:active}

osd: 3 osds: 3 up, 3 in

data:

pools: 2 pools, 256 pgs

objects: 21 objects, 2246 bytes

usage: 3080 MB used, 58347 MB / 61428 MB avail

pgs: 256 active+clean

1.4 客户端挂载

1.4.1 用户空间挂载 CEPH 文件系统

在linux-node3上执行如下配置

[ceph@linux-node3 ceph]$ sudo yum install -y ceph-fuse.x86_6

[ceph@linux-node3 ceph]$ sudo scp root@linux-node1:/home/ceph/my-cluster/ceph.conf /etc/ceph/ceph.conf

[ceph@linux-node3 ceph]$ sudo scp root@linux-node1:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring

[ceph@linux-node3 ceph]$ sudo mkdir /home/ceph/cephfs

[ceph@linux-node3 ceph]$ sudo ceph-fuse -m 192.168.100.151:6789 /home/ceph/cephfs

1.4.2 在linux-node1上执行如下配置

[ceph@linux-node1 ceph]$ sudo yum install -y ceph-fuse.x86_6

[ceph@linux-node1 ceph]$ sudo scp root@linux-node1:/home/ceph/my-cluster/ceph.conf /etc/ceph/ceph.conf

[ceph@linux-node1 ceph]$ sudo scp root@linux-node1:/etc/ceph/ceph.client.admin.keyring /etc/ceph/ceph.client.admin.keyring

[ceph@linux-node1 ceph]$ sudo mkdir /home/ceph/cephfs

[ceph@linux-node1 ceph]$ sudo ceph-fuse -m 192.168.100.151:6789 /home/ceph/cephfs

1.5 用内核驱动挂载 CEPH 文件系统

1.5.1 要挂载 Ceph 文件系统，如果你知道监视器IP地址可以用 mount 命令、或者用 mount.ceph工具来自动解析监视器IP地址

# sudo mkdir /mnt/mycephfs

# sudo mount -t ceph 192.168.100.151:6789:/ /mnt/mycephfs

1.5.2 要挂载启用了 cephx 认证的 Ceph 文件系统，你必须指定用户名、密钥

# sudo mount -t ceph 192.168.100.151:6789:/ /mnt/mycephfs -o name=admin,secret=AQATSKdNGBnwLhAAnNDKnH65FmVKpXZJVasUeQ==

1.5.3 前述用法会把密码遗留在 Bash 历史里，更安全的方法是从文件读密码。例如

# sudo mount -t ceph 192.168.100.151:6789:/ /mnt/mycephfs -o name=admin,secretfile=/etc/ceph/admin.secret

1.5.4 要卸载 Ceph 文件系统，可以用 unmount 命令

# sudo umount /mnt/mycephfs

1.6 CEPHFS配额管理

1.6.1 创建该文件目录的磁盘配额为100MB

[ceph@linux-node1 ~]$ sudo setfattr -n ceph.quota.max_bytes -v 100000000 /home/ceph/cephfs/k8s

1.6.2 查看磁盘配额

[ceph@linux-node1 ~]$ sudo getfattr -n ceph.quota.max_bytes /home/ceph/cephfs/k8s

getfattr: Removing leading '/' from absolute path names

# file: home/ceph/cephfs/k8s

ceph.quota.max_bytes="100000000"

1.6.3 测试配额效果

[ceph@linux-node1 k8s]$ dd if=/dev/zero of=1.txt bs=1M count=200

dd: failed to open ‘1.txt’: Permission denied

[ceph@linux-node1 k8s]$ sudo dd if=/dev/zero of=1.txt bs=1M count=200

dd: error writing ‘1.txt’: Disk quota exceeded

105+0 records in

104+0 records out

109314048 bytes (109 MB) copied, 3.52606 s, 31.0 MB/

1.6.4 再一次测试，发现无法写入了。

[ceph@linux-node1 k8s]$ sudo dd if=/dev/zero of=2.txt bs=1M count=200

dd: error writing ‘2.txt’: Disk quota exceeded

1+0 records in

0+0 records out

0 bytes (0 B) copied, 0.00256971 s, 0.0 kB/

1.6.5 删除配额

[ceph@linux-node1 k8s]$ sudo setfattr -n ceph.quota.max_bytes -v 0 /home/ceph/cephfs/k8s

[ceph@linux-node1 k8s]$ sudo getfattr -n ceph.quota.max_bytes /home/ceph/cephfs/k8s

/home/ceph/cephfs/k8s: ceph.quota.max_bytes: No such attribute

1.6.6 删除已经创建的块设备

[root@linux-node1 ~]# rbd ls list -p volumes

rbd_test

volume-5e0b9cf5-6ebe-4641-8436-c3451af51eb9

[root@linux-node1 ~]# rbd unmap volumes/rbd_test

[root@linux-node1 ~]# rbd showmapped

[root@linux-node1 ~]# rbd remove volumes/rbd_test

Removing image: 100% complete...done.

[root@linux-node1 ~]# rbd ls list -p volumes

volume-5e0b9cf5-6ebe-4641-8436-c3451af51eb9

最后编辑于：2019.11.22 19:18:29

人面猴
序言：七十年代末，一起剥皮案震惊了整个滨河市，随后出现的几起案子，更是在滨河造成了极大的恐慌，老刑警刘岩，带你破解...
沈念sama阅读 204,793评论 6赞 478
死咒
序言：滨河连续发生了三起死亡事件，死亡现场离奇诡异，居然都是意外死亡，警方通过查阅死者的电脑和手机，发现死者居然都...
沈念sama阅读 87,567评论 2赞 381
救了他两次的神仙让他今天三更去死
文/潘晓璐我一进店门，熙熙楼的掌柜王于贵愁眉苦脸地迎上来，“玉大人，你说我怎么就摊上这事。” “怎么了？”我有些...
开封第一讲书人阅读 151,342评论 0赞 338
道士缉凶录：失踪的卖姜人
文/不坏的土叔我叫张陵，是天一观的道长。经常有香客问我，道长，这世上最难降的妖魔是什么？我笑而不...
开封第一讲书人阅读 54,825评论 1赞 277
港岛之恋（遗憾婚礼）
正文为了忘掉前任，我火速办了婚礼，结果婚礼上，老公的妹妹穿的比我还像新娘。我一直安慰自己，他们只是感情好，可当我...
茶点故事阅读 63,814评论 5赞 368
恶毒庶女顶嫁案：这布局不是一般人想出来的
文/花漫我一把揭开白布。她就那样静静地躺着，像睡着了一般。火红的嫁衣衬着肌肤如雪。梳的纹丝不乱的头发上，一...
开封第一讲书人阅读 48,680评论 1赞 281
城市分裂传说
那天，我揣着相机与录音，去河边找鬼。笑死，一个胖子当着我的面吹牛，可吹牛的内容都是我干的。我是一名探鬼主播，决...
沈念sama阅读 38,033评论 3赞 399
双鸳鸯连环套：你想象不到人心有多黑
文/苍兰香墨我猛地睁开眼，长吁一口气：“原来是场噩梦啊……” “哼！你这毒妇竟也来了？” 一声冷哼从身侧响起，我...
开封第一讲书人阅读 36,687评论 0赞 258
万荣杀人案实录
序言：老挝万荣一对情侣失踪，失踪者是张志新（化名）和其女友刘颖，没想到半个月后，有当地人在树林里发现了一具尸体，经...
沈念sama阅读 42,175评论 1赞 300
护林员之死
正文独居荒郊野岭守林人离奇死亡，尸身上长有42处带血的脓包…… 初始之章·张勋以下内容为张勋视角年9月15日...
茶点故事阅读 35,668评论 2赞 321
白月光启示录
正文我和宋清朗相恋三年，在试婚纱的时候发现自己被绿了。大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
茶点故事阅读 37,775评论 1赞 332
活死人
序言：一个原本活蹦乱跳的男人离奇死亡，死状恐怖，灵堂内的尸体忽然破棺而出，到底是诈尸还是另有隐情，我是刑警宁泽，带...
沈念sama阅读 33,419评论 4赞 321
日本核电站爆炸内幕
正文年R本政府宣布，位于F岛的核电站，受9级特大地震影响，放射性物质发生泄漏。R本人自食恶果不足惜，却给世界环境...
茶点故事阅读 39,020评论 3赞 307
男人毒药：我在死后第九天来索命
文/蒙蒙一、第九天我趴在偏房一处隐蔽的房顶上张望。院中可真热闹，春花似锦、人声如沸。这庄子的主人今日做“春日...
开封第一讲书人阅读 29,978评论 0赞 19
一桩弑父案，背后竟有这般阴谋
文/苍兰香墨我抬头看了看天上的太阳。三九已至，却和暖如春，着一层夹袄步出监牢的瞬间，已是汗流浃背。一阵脚步声响...
开封第一讲书人阅读 31,206评论 1赞 260
情欲美人皮
我被黑心中介骗来泰国打工，没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留，地道东北人。一个月前我还...
沈念sama阅读 45,092评论 2赞 351
代替公主和亲
正文我出身青楼，却偏偏与公主长得像，于是被迫代替她去往敌国和亲。传闻我的和亲对象是个残疾皇子，可洞房花烛夜当晚...
茶点故事阅读 42,510评论 2赞 343

ceph分布式存储配置

推荐阅读更多精彩内容