四川群奸门事件,神风终极战士,cf新年广场遁地
主机 | ip | 磁盘 | 备注 |
k8smaster01 | 172.24.8.71 | kubernetes master节点 | |
k8smaster02 | 172.24.8.72 | kubernetes master节点 | |
k8smaster03 | 172.24.8.73 | kubernetes master节点 | |
k8snode01 | 172.24.8.74 | sdb | kubernetes node节点 glusterfs节点 |
k8snode02 | 172.24.8.75 | sdb | kubernetes node节点 glusterfs节点 |
k8snode03 | 172.24.8.76 | sdb | kubernetes node节点 glusterfs节点 |
磁盘规划
1 172.24.8.71 k8smaster01 2 172.24.8.72 k8smaster02 3 172.24.8.73 k8smaster03 4 172.24.8.74 k8snode01 5 172.24.8.75 k8snode02 6 172.24.8.76 k8snode03
1 [root@k8snode01 ~]# fdisk /dev/sdb -l #检查sdb是否为裸磁盘
1 [root@k8snode01 ~]# yum -y install centos-release-gluster 2 [root@k8snode01 ~]# yum -y install glusterfs-fuse #安装glusterfs-fuse
1 [root@k8snode01 ~]# cat > /etc/sysconfig/modules/glusterfs.modules <<eof 2 #!/bin/bash 3 4 for kernel_module in dm_snapshot dm_mirror dm_thin_pool;do 5 /sbin/modinfo -f filename \${kernel_module} > /dev/null 2>&1 6 if [ \$? -eq 0 ]; then 7 /sbin/modprobe \${kernel_module} 8 fi 9 done; 10 eof 11 [root@k8snode01 ~]# chmod +x /etc/sysconfig/modules/glusterfs.modules 12 [root@k8snode01 ~]# lsmod |egrep "dm_snapshot|dm_mirror|dm_thin_pool" #所有glusterfs node节点检查
1 [root@k8smaster01 ~]# kubectl label nodes k8snode01 storagenode=glusterfs 2 [root@k8smaster01 ~]# kubectl label nodes k8snode02 storagenode=glusterfs 3 [root@k8smaster01 ~]# kubectl label nodes k8snode03 storagenode=glusterfs
1 [root@k8smaster01 ~]# yum -y install git 2 [root@k8smaster01 ~]# git clone https://github.com/gluster/gluster-kubernetes.git
1 [root@k8smaster01 ~]# cd gluster-kubernetes/deploy/ 2 [root@k8smaster01 deploy]# cp topology.json.sample topology.json 3 [root@k8smaster01 deploy]# vi topology.json
1 { 2 "clusters": [ 3 { 4 "nodes": [ 5 { 6 "node": { 7 "hostnames": { 8 "manage": [ 9 "k8snode01" 10 ], 11 "storage": [ 12 "172.24.8.74" 13 ] 14 }, 15 "zone": 1 16 }, 17 "devices": [ 18 "/dev/sdb" 19 ] 20 }, 21 { 22 "node": { 23 "hostnames": { 24 "manage": [ 25 "k8snode02" 26 ], 27 "storage": [ 28 "172.24.8.75" 29 ] 30 }, 31 "zone": 1 32 }, 33 "devices": [ 34 "/dev/sdb" 35 ] 36 }, 37 { 38 "node": { 39 "hostnames": { 40 "manage": [ 41 "k8snode03" 42 ], 43 "storage": [ 44 "172.24.8.76" 45 ] 46 }, 47 "zone": 1 48 }, 49 "devices": [ 50 "/dev/sdb" 51 ] 52 } 53 ] 54 } 55 ] 56 }
1 [root@k8smaster01 deploy]# cp heketi.json.template heketi.json 2 [root@k8smaster01 deploy]# vi heketi.json 3 { 4 "_port_comment": "heketi server port number", 5 "port" : "8080", 6 7 "_use_auth": "enable jwt authorization. please enable for deployment", 8 "use_auth" : true, #开启用户认证 9 10 "_jwt" : "private keys for access", 11 "jwt" : { 12 "_admin" : "admin has access to all apis", 13 "admin" : { 14 "key" : "admin123" #管理员密码 15 }, 16 "_user" : "user only has access to /volumes endpoint", 17 "user" : { 18 "key" : "xianghy" #用户密码 19 } 20 }, 21 22 "_glusterfs_comment": "glusterfs configuration", 23 "glusterfs" : { 24 25 "_executor_comment": "execute plugin. possible choices: mock, kubernetes, ssh", 26 "executor" : "${heketi_executor}", #本实验使用kubernetes方式 27 28 "_db_comment": "database file name", 29 "db" : "/var/lib/heketi/heketi.db", #heketi数据存储 30 31 "kubeexec" : { 32 "rebalance_on_expansion": true 33 }, 34 35 "sshexec" : { 36 "rebalance_on_expansion": true, 37 "keyfile" : "/etc/heketi/private_key", 38 "port" : "${ssh_port}", 39 "user" : "${ssh_user}", 40 "sudo" : ${ssh_sudo} 41 } 42 }, 43 44 "backup_db_to_kube_secret": false 45 }
1 [root@k8smaster01 deploy]# vi gk-deploy 2 924 #heketi_pod=$(${cli} get pod --no-headers --show-all --selector="heketi" | awk '{print $1}') 3 925 heketi_pod=$(${cli} get pod --no-headers --selector="heketi" | awk '{print $1}')
1 [root@vpn ~]# docker pull gluster/gluster-centos:latest 2 [root@vpn ~]# docker pull heketi/heketi:dev 3 [root@vpn ~]# docker save -o gluster_latest.tar gluster/gluster-centos:latest 4 [root@vpn ~]# docker save -o heketi_dev.tar heketi/heketi:dev 5 [root@k8snode01 ~]# docker load -i gluster_latest.tar 6 [root@k8snode01 ~]# docker load -i heketi_dev.tar 7 [root@k8snode01 ~]# docker images 8
1 [root@k8smaster01 deploy]# ./gk-deploy -h #查看部署参数 2 [root@k8smaster01 deploy]# kubectl create ns heketi #建议部署在独立的namespace中 3 [root@k8smaster01 deploy]# ./gk-deploy -g -n heketi topology.json --admin-key admin123 --user-key xianghy 4 …… 5 do you wish to proceed with deployment? 6 7 [y]es, [n]o? [default: y]: y
1 [root@k8smaster01 deploy]# ./gk-deploy --abort --admin-key admin123 --user-key xianghy -y -n heketi 2 [root@k8smaster01 deploy]# kubectl delete -f kube-templates/ -n heketi
1 [root@k8snode01 ~]# dmsetup ls 2 [root@k8snode01 ~]# dmsetup remove_all 3 [root@k8snode01 ~]# rm -rf /var/log/glusterfs/ 4 [root@k8snode01 ~]# rm -rf /var/lib/heketi 5 [root@k8snode01 ~]# rm -rf /var/lib/glusterd/ 6 [root@k8snode01 ~]# rm -rf /etc/glusterfs/ 7 [root@k8snode01 ~]# dd if=/dev/zero of=/dev/sdb bs=512k count=1 8 [root@k8snode01 ~]# wipefs -af /dev/sdb
1 [root@k8smaster01 ~]# kubectl get nodes --show-labels | grep -e 'name|node' 2 [root@k8smaster01 ~]# kubectl get all -n heketi
1 [root@k8smaster01 ~]# kubectl get pods -o wide -n heketi
1 [root@k8smaster01 ~]# kubectl exec -it heketi-65f4555d74-72hrf -n heketi -- heketi-cli cluster list --user admin --secret admin123 #集群列表 2 [root@k8smaster01 ~]# kubectl -n heketi exec -ti heketi-65f4555d74-72hrf /bin/bash [root@heketi-65f4555d74-72hrf /]# heketi-cli cluster list --user admin --secret admin123 #进入heketi容器查看 3 [root@k8smaster01 ~]# curl http://10.254.111.219:8080/hello 4 hello from heketi
1 [root@k8smaster01 ~]# yum -y install centos-release-gluster 2 [root@k8smaster01 ~]# yum -y install heketi-client
1 [root@k8smaster01 ~]# echo "export heketi_cli_server=http://$(kubectl get svc heketi -n heketi -o go-template='{{.spec.clusterip}}'):8080" >> /etc/profile.d/heketi.sh 2 [root@k8smaster01 ~]# echo "alias heketi-cli='heketi-cli --user admin --secret admin123'" >> ~/.bashrc 3 [root@k8smaster01 ~]# source /etc/profile.d/heketi.sh 4 [root@k8smaster01 ~]# source ~/.bashrc 5 [root@k8smaster01 ~]# echo $heketi_cli_server 6 http://heketi:8080
1 [root@k8smaster01 ~]# heketi-cli cluster list 2 clusters: 3 id:67004a06fbcb4fa525bcec1fbaa9ef2d [file][block] 4 [root@k8smaster01 ~]# heketi-cli cluster info 67004a06fbcb4fa525bcec1fbaa9ef2d #集群详细信息 5 cluster id: 67004a06fbcb4fa525bcec1fbaa9ef2d 6 nodes: 7 40cdd4c1d0c389939193d6dea3c5bfe8 8 62873c54cf61025fda91e6d44433378b 9 d48986357840d28653304e7170599da5 10 volumes: 11 5f15f201d623e56b66af56313a1975e7 12 block: true 13 14 file: true 15 [root@k8smaster01 ~]# heketi-cli topology info 67004a06fbcb4fa525bcec1fbaa9ef2d #查看拓扑信息 16 [root@k8smaster01 ~]# heketi-cli node list #查看所有node 17 id:40cdd4c1d0c389939193d6dea3c5bfe8 cluster:67004a06fbcb4fa525bcec1fbaa9ef2d 18 id:62873c54cf61025fda91e6d44433378b cluster:67004a06fbcb4fa525bcec1fbaa9ef2d 19 id:d48986357840d28653304e7170599da5 cluster:67004a06fbcb4fa525bcec1fbaa9ef2d 20 [root@k8smaster01 ~]# heketi-cli node info 40cdd4c1d0c389939193d6dea3c5bfe8 #node节点信息 21 [root@k8smaster01 ~]# heketi-cli volume create --size=2 --replica=2 #默认为3副本的replica模式
1 [root@k8smaster01 ~]# heketi-cli volume list #列出所有卷 2 [root@k8smaster01 ~]# heketi-cli volume info fc296ab350dcc36e00dd3b3643a04645 #卷信息 3 [root@k8smaster01 ~]# heketi-cli volume delete fc296ab350dcc36e00dd3b3643a04645 #删除卷
1 [root@k8smaster01 ~]# echo -n "admin123" | base64 #将密码转换为64位编码 2 ywrtaw4xmjm= 3 [root@k8smaster01 ~]# mkdir -p heketi 4 [root@k8smaster01 ~]# cd heketi/ 5 [root@k8smaster01 ~]# vi heketi-secret.yaml #创建用于保存密码的secret 6 apiversion: v1 7 kind: secret 8 metadata: 9 name: heketi-secret 10 namespace: heketi 11 data: 12 # base64 encoded password. e.g.: echo -n "mypassword" | base64 13 key: ywrtaw4xmjm= 14 type: kubernetes.io/glusterfs 15 [root@k8smaster01 heketi]# kubectl create -f heketi-secret.yaml #创建heketi 16 [root@k8smaster01 heketi]# kubectl get secrets -n heketi 17 name type data age 18 default-token-6n746 kubernetes.io/service-account-token 3 144m 19 heketi-config-secret opaque 3 142m 20 heketi-secret kubernetes.io/glusterfs 1 3m1s 21 heketi-service-account-token-ljlkb kubernetes.io/service-account-token 3 143m 22 [root@kubenode1 heketi]# vim gluster-heketi-storageclass.yaml #正式创建storageclass 23 apiversion: storage.k8s.io/v1 24 kind: storageclass 25 metadata: 26 name: gluster-heketi-storageclass 27 parameters: 28 resturl: "http://10.254.111.219:8080" 29 clusterid: "67004a06fbcb4fa525bcec1fbaa9ef2d" 30 restauthenabled: "true" #若heketi开启认证此处也必须开启auth认证 31 restuser: "admin" 32 secretname: "heketi-secret" #name/namespace与secret资源中定义一致 33 secretnamespace: "heketi" 34 volumetype: "replicate:3" 35 provisioner: kubernetes.io/glusterfs 36 reclaimpolicy: delete 37 [root@k8smaster01 heketi]# kubectl create -f gluster-heketi-storageclass.yaml
1 [root@k8smaster01 heketi]# kubectl get storageclasses #查看确认 2 name provisioner age 3 gluster-heketi-storageclass kubernetes.io/glusterfs 85s 4 [root@k8smaster01 heketi]# kubectl describe storageclasses gluster-heketi-storageclass
1 [root@k8smaster01 heketi]# vi gluster-heketi-pvc.yaml 2 apiversion: v1 3 kind: persistentvolumeclaim 4 metadata: 5 name: gluster-heketi-pvc 6 annotations: 7 volume.beta.kubernetes.io/storage-class: gluster-heketi-storageclass 8 spec: 9 accessmodes: 10 - readwriteonce 11 resources: 12 requests: 13 storage: 1gi
1 [root@k8smaster01 heketi]# kubectl create -f gluster-heketi-pvc.yaml -n heketi 2 [root@k8smaster01 heketi]# kubectl get pvc -n heketi 3 [root@k8smaster01 heketi]# kubectl describe pvc gluster-heketi-pvc -n heketi 4 [root@k8smaster01 heketi]# kubectl get pv -n heketi 5 [root@k8smaster01 heketi]# kubectl describe pv pvc-ca949559-094a-11ea-8b3c-000c29fa7a79 -n heketi
1 [root@k8smaster01 heketi]# kubectl describe endpoints glusterfs-dynamic-ca949559-094a-11ea-8b3c-000c29fa7a79 -n heketi
1 [root@k8smaster01 ~]# kubectl get pod -n heketi 2 [root@k8smaster01 ~]# kubectl exec -ti glusterfs-b854k -n heketi -- lsblk #glusterfs节点查看 3 [root@k8smaster01 ~]# kubectl exec -ti glusterfs-b854k -n heketi -- df -ht #glusterfs节点查看 4 [root@k8smaster01 ~]# kubectl exec -ti glusterfs-b854k -n heketi -- gluster volume list 5 [root@k8smaster01 ~]# kubectl exec -ti glusterfs-b854k -n heketi -- gluster volume info vol_29ba6f9665522ad5893412e61799a433 #glusterfs节点查看
1 [root@xxx ~]# yum -y install centos-release-gluster 2 [root@xxx ~]# yum -y install glusterfs-fuse #安装glusterfs-fuse
1 [root@k8smaster01 heketi]# vi gluster-heketi-pod.yaml 2 kind: pod 3 apiversion: v1 4 metadata: 5 name: gluster-heketi-pod 6 spec: 7 containers: 8 - name: gluster-heketi-container 9 image: busybox 10 command: 11 - sleep 12 - "3600" 13 volumemounts: 14 - name: gluster-heketi-volume #必须和volumes中name一致 15 mountpath: "/pv-data" 16 readonly: false 17 volumes: 18 - name: gluster-heketi-volume 19 persistentvolumeclaim: 20 claimname: gluster-heketi-pvc #必须和5.3创建的pvc中的name一致 21 [root@k8smaster01 heketi]# kubectl create -f gluster-heketi-pod.yaml -n heketi #创建pod
1 [root@k8smaster01 ~]# kubectl get pod -n heketi | grep gluster-heketi 2 gluster-heketi-pod 1/1 running 0 4m58s 3 [root@k8smaster01 ~]# kubectl exec -it gluster-heketi-pod /bin/sh -n heketi #进入pod写入测试文件 4 / # cd /pv-data/ 5 /pv-data # echo "this is a file!" >> a.txt 6 /pv-data # echo "this is b file!" >> b.txt 7 /pv-data # ls 8 a.txt b.txt 9 [root@k8smaster01 ~]# kubectl exec -it gluster-heketi-pod -n heketi -- df -h #查看所挂载的glusterfs
1 [root@k8smaster01 ~]# kubectl get pods -n heketi -o wide #查看对应的glusterfs node
1 [root@k8smaster01 ~]# kubectl exec -ti glusterfs-b854k -n heketi -- cat /var/lib/heketi/mounts/vg_2c7a02d1b1b7c1f165283b6691062102/brick_16e37a18a5e5fd40e14338ba78d99565/brick/a.txt 2 this is a file!
1 [root@k8smaster01 ~]# cd heketi/ 2 [root@k8smaster01 heketi]# kubectl delete -f gluster-heketi-pod.yaml -n heketi 3 [root@k8smaster01 heketi]# kubectl delete -f gluster-heketi-pvc.yaml 4 [root@k8smaster01 heketi]# kubectl get pvc -n heketi 5 [root@k8smaster01 heketi]# kubectl get pv -n heketi 6 [root@k8smaster01 heketi]# kubectl exec -ti glusterfs-b854k -n heketi gluster volume list | grep gluster
如对本文有疑问,请在下面进行留言讨论,广大热心网友会与你互动!! 点击进行留言回复
网友评论