1)disable iptables&selinux

1
2
3
4
5
6
7
8
9
10
11
12
13
echo -n "正在配置iptables防火墙……"
systemctl stop firewalld > /dev/null 2>&1
systemctl disable firewalld  > /dev/null 2>&1
if [ $? -eq 0 ];then
echo -n "Iptables防火墙初始化完毕！"
fi

echo -n "正在关闭SELinux……"
setenforce 0 > /dev/null 2>&1
sed -i '/^SELINUX=/s/=.*/=disabled/' /etc/selinux/config
if [ $? -eq 0 ];then
    echo -n "SELinux初始化完毕！"
fi
  1. set hostname as ceph
    1
    2
    3
    4
    HOSTNAME=ceph
    hostnamectl set-hostname ceph
    IP=`ip route |grep src|grep metric|awk -F" " '{ print $9 }'`
    echo "$IP $HOSTNAME" >>/etc/hosts
  2. install epel.repo
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    yum install -y epel-release
    cat <<EOF > /etc/yum.repos.d/ceph.repo
    [Ceph]
    name=Ceph packages for $basearch
    baseurl=http://mirrors.163.com/ceph/rpm-jewel/el7/x86_64
    enabled=1
    gpgcheck=1
    type=rpm-md
    gpgkey=http://mirrors.163.com/ceph/keys/release.asc
    priority=1

    [Ceph-noarch]
    name=Ceph noarch packages
    baseurl=http://mirrors.163.com/ceph/rpm-jewel/el7/noarch
    enabled=1
    gpgcheck=1
    type=rpm-md
    gpgkey=http://mirrors.163.com/ceph/keys/release.asc
    priority=1

    [ceph-source]
    name=Ceph source packages
    baseurl=http://mirrors.163.com/ceph/rpm-jewel/el7/SRPMS
    enabled=1
    gpgcheck=1
    type=rpm-md
    gpgkey=http://mirrors.163.com/ceph/keys/release.asc
    priority=1
    EOF
    4)update system & install ceph-deploy
    1
    yum update -y &&yum clean all &&yum -y install ceph-deploy
  3. 设置本机密匙
    1
    2
      ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
    cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
    6)ceph服务初始化
    1
    2
    3
    yum clean all &&yum -y install ceph-deploy
    mkdir /etc/ceph &&cd /etc/ceph
    ceph-deploy new ceph
    7)修改配置文件
    1
    2
    3
    cp ceph.conf ceph.conf.bak
    sed -i 's@^$@osd_pool_default_size = 1@g' ceph.conf
    echo "mon_pg_warn_max_per_osd = 1000" >> /etc/ceph/ceph.conf
    8)安装ceph
    1
    ceph-deploy install ceph
    9)创建monitor服务
    1
    2
    ceph-deploy mon create ceph
    ceph-deploy  gatherkeys ceph
  4. 准备osd
    1
    2
    3
    4
    5
    6
    7
    8
    9
    mkfs.xfs  /dev/sdb
    mkdir -p /var/local/osd
    mount /dev/sdb  /var/local/osd/
    chown -R ceph:ceph /var/local/osd*      #创建osd
    ceph-deploy osd prepare ceph:/var/local/osd  #激活osd
    ceph-deploy osd activate ceph:/var/local/osd
    chown -R ceph:ceph /var/local/osd*   #有些同学可能会忘记配置目录权限引起激活osd失败 
    #查看状态：###
    ceph-deploy osd list ceph
    11）修改配置文件权限
    1
    2
    ceph-deploy admin ceph
    chmod +r /etc/ceph/*
    12）部署mds服务
    1
    2
    ceph-deploy mds create ceph
    ceph mds stat
    13）创建ceph文件系统
    1
    2
    3
    4
    5
    ceph fs ls
    ceph osd pool create cephfs_data 128
    ceph osd pool create cephfs_metadata 128
    ceph fs new cephfs cephfs_metadata cephfs_data
    ceph fs ls
    14）挂载Ceph文件系统
    1
    2
    3
    4
    5
    6
    mkdir /ceph
    yum install -y ceph-fuse
    IP=`ip route |grep src|grep metric|awk -F" " '{ print $9 }'`
    ceph-fuse -m $IP:7480/ /ceph
    #ceph-fuse -m $IP:6789/ /ceph
    df -Th
    15）查看ceph状态
    1
    2
    3
    4
    5
    ceph mon stat
    ceph osd stat
    ceph osd tree  #显示crush图
    ceph pg stat
    set password for root@localhost = password('S@Aj#XrpS114y!9$');  #设置mysql密码

##部署ceph对象存储-bucket
1)安装ceph-radosgw

1
yum install ceph-radosgw

2)部署rgw

1
ceph-deploy rgw create ceph  #需在/etc/ceph目录下执行

3)如果要修改为80端口，可修改配置文件 重启

1
2
3
4
5
 vim /etc/ceph/ceph.conf
 [client.rgw.ceph-node1]
rgw_frontends = "civetweb port=80"
sudo systemctl restart ceph-radosgw@rgw.ceph.service
netstat -tnlp |grep 7480

4)创建池

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
wget https://raw.githubusercontent.com/aishangwei/ceph-demo/master/ceph-deploy/rgw/pool
wget  https://raw.githubusercontent.com/aishangwei/ceph-demo/master/ceph-deploy/rgw/create_pool.sh

cat create_pool.sh
#!/bin/bash

PG_NUM=250
PGP_NUM=250
SIZE=3

for i in `cat /etc/ceph/pool`
        do
        ceph osd pool create $i $PG_NUM
        ceph osd pool set $i size $SIZE
        done

for i in `cat /etc/ceph/pool`
        do
        ceph osd pool set $i pgp_num $PGP_NUM
        done

chmod +x create_pool.sh
./create_pool.sh

5)测试是否能访问ceph集群

1
2
3
ls -l /var/lib/ceph/
cp /var/lib/ceph/radosgw/ceph-rgw.ceph/keyring ./
ceph -s -k keyring --name client.rgw.ceph

6)使用S3 API访问ceph对象存储
创建radosgw用户

1
radosgw-admin user create --uid=radosgw --display-name="radosgw"

7)安装s3cmd客户端

1
2
3
4
5
6
7
yum install -y s3cmd
s3cmd --configure  //注意该Access Key填写radosgw的access_key  Secret Key同样
编辑s3配置文件：cat .s3cfg
创建桶并放入文件
s3cmd mb s3://devops
s3cmd ls
s3cmd put /etc/hosts s3://devops

8)使用Swift API访问ceph对象存储

1
2
3
4
5
6
7
radosgw-admin subuser create --uid=radosgw --subuser=radosgw:swift --access=full
yum install python-pip -y
pip install --upgrade python-swiftclient
swift -A http://ceph:7480/auth/1.0 -U radosgw:swift -K 6Dz8f8ubJDNhr57jV5tM0xeFSzBxXN1MzK59qdMh list
swift -A http://ceph:7480/auth/1.0 -U radosgw:swift -K 6Dz8f8ubJDNhr57jV5tM0xeFSzBxXN1MzK59qdMh list
swift -A http://ceph:7480/auth/1.0 -U radosgw:swift -K 6Dz8f8ubJDNhr57jV5tM0xeFSzBxXN1MzK59qdMh post second-bucket
swift -A http://ceph:7480/auth/1.0 -U radosgw:swift -K 6Dz8f8ubJDNhr57jV5tM0xeFSzBxXN1MzK59qdMh list

9)最后验证

1
2
3
4
s3cmd ls
2019-02-15 07:45  s3://devops
2019-02-15 08:18  s3://second-bucket
s3cmd ls s3://devops/20190619/