教育436实验手册

双节点HA与仲裁磁 LVM高级扩 HA- 附 CPUVT-xCPUVT-xEPTvmwareworkstation9之后，似乎取消了模拟方式，想要实现虚拟化嵌套，CPU必须支持EPT。EPT是一种硬件辅助的内存虚拟化方式，内存虚拟化实际就是进行地址转换从客户都已经支持EPT指令。没有AMDCPU，所以AMD情况不了解。需要CPU支持VT-xEPT指令集。有关虚拟化定义，猛击此处查看百科的定义（打不开自行寻找方式vmwareworkstation/virtualbox/kvm可以使用RH401课程的instructor，在server上配置好DNS服务，域 ISCSIiqn:iSCSIQualifiediqn格式：iqn.YYYY-MM.reversed target:iscsi服务[root@node4~]#yum-y[root@node4~]#yum-yinstallscsi-target-#在sda2G10G10G分区设为LVM[root@node4~]#fdisk-lDisk/dev/sda5:2152255heads,63sectors/track,261Disk/dev/sda6:10.7GB,10742183424255heads,63sectors/track,1305Writingphysicalvolumedatatodisk[root@node4~]#vgcreatevgsrv/dev/sda6[root@node4~]#vgs #PV#LV#SN VSize 0wz--n-10.00g10.00g[root@node4~]#[root@node4~]#lvcreate-L1G-nexample[root@node4~]#lvs LSizeOriginSnap%MoveLogCopy%Convertexamplevgsrv-wi-a-1.00g[root@node4~]#lllrwxrwxrwx1rootroot7Aug603:07/dev/vgsrv/example->../dm-[root@node4~]#vim#38 StartingSCSItarget [OK[root@node4~]#tgt-adminshowNoactionspecified.[root@node4~]#tgt-admin--Target1: Systeminformation:Driver:iscsiI_TnexusLUNLUN: 能力，所以size是SCSIID:SCSISN:Size:0MB,Blocksize:Online:YesReadonly:NoBackingstoretype:nullBackingstoreflags:LUN:Type:diskSCSIID:IETSCSISN:Online:YesReadonly:NoBackingstoretype:Backingstoreflags:ACLinformation:[root@node1~]#yum-yinstalliscsi-initiator-[root@node1~]#iscsiadm-mdiscovery-tsendtargets-p [OK [root@node1~]#[root@node1~]#iscsiadm-mnode- Logginginto[iface:default,target: .example:exampletarget,portal:,3260]Loginto[iface:default,target: [root@node1~]##查看iscsi挂接情况三种的方式[root@node1~fdiskldev/sdb[root@node1~]#[root@node1~]#ls/dev/disk/by-path/-total lrwxrwxrwx1rootroot9Aug602:03pci-0000:00:10.0-scsi-0:0:0:0->../../sdalrwxrwxrwx1rootroot10Aug602:03pci-0000:00:10.0-scsi-0:0:0:0-part1->../.lrwxrwxrwx1rootroot10Aug602:03pci-0000:00:10.0-scsi-0:0:0:0-part2->../.lrwxrwxrwx1rootroot10Aug602:03pci-0000:00:10.0-scsi-0:0:0:0-part3->.././sda3[root@node1~]#[root@node1~iscsiadmmsession # CurrentPortal:4:3260,1Iface IfaceNetdev:<empty>SID:iSCSISessionState:LOGGED_INInternaliscsidSessionState:NOIface.Initiatorname实际工作中，这里最好修改下，修改成自己主机名相关，这样方便于管理，修改##登出[root@node1~]#iscsiadm-mnode- Loggingoutofsession[sid:1,target: Logoutof[sid:1, .example:exampletarget,portal:,3260]#[root@node1~]#rm-rf#停止iSCSIinitiator [OK [OK#修改[root@node1~]#cat [root@node1~]#iscsiadm-mdiscovery-tsendtargets-p#[OK [root@node1~]#iscsiadm-msession-P1 CurrentPortal:Iface IfaceHWaddress:<empty>IfaceNetdev:<empty>SID:iSCSISessionState:LOGGED_INInternaliscsidSessionState:NO#backing #direct-store 网络上千篇一律的说，这个是用于一个整个磁盘，但是没有人说这个##backing #direct-store 网络上千篇一律的说，这个是用于一个整个磁盘，但是没有人说这个#参数实际上在虚拟机里无法生效的，如果想用direct-store，用实体机测试。#cat backing-store/dev/sda5[root@node1~]#iscsiadm-mdiscovery-tsendtargets-p:3260 [root@node1[root@node1~]#iscsiadm-mnode-Logginginto[iface:default,target:Loginto[iface:default,.example.node4:sdbtarget,portal:.example.node4:sdbtarget,portal:##vendor-idRHCAIncinitiation端可以看到LUNscsi_sns01#initiator-address #配置可以挂接这个LUN的客户端，如果不initiator-address #配置，则表示，所有机器均可挂接initiator-address#修改配置完成之后重启服务，但是此时LUN已经被# SCSItargetdaemon:initiatorsstillconnected StartingSCSItargetdaemon: [root@node1~]#iscsiadm-mnode- Loggingoutof[root@node1~]#iscsiadm-mnode- #node4 SCSItarget [OKStartingSCSItarget [OKnode1[root@node1~]#iscsiadm-mnode- .example:exampletarget- #但是这样太麻烦了！客户端有两个init.d ，/etc/init.d/iscsi和iscsid，iscsi是控制LUN的卸载的，所以就只要stopiscsi即可node1停止iscsi [OKnode4[root@node4~]# SCSItargetdaemon: [OK]StartingSCSItarget [OKLUN:SCSISN:s01Online:YesReadonly:NoBackingstoretype:Backingstoreflags:AccountACLnode1 [OKParted但是实际工作中，如果使用到专业设备，往往单个LUN是远超过2T，那就要使用GPT分区，fdisk无法对gpt分区，此时需要使用parted。11049kB200MB2 300MB11049kB200MBiscsi2 300MBUDEVudev#在node4上建立一个lvm卷，同时打开udevadm[root@node4~]##在node4上建立一个lvm卷，同时打开udevadm[root@node4~]#lvcreate-L3G-nudevtestvgsrvLogicalvolume"udevtest"created[root@node4[root@node4~]#udevadmmonitorwillprintthereceivedeventsUDEV-theeventwhichudevsendsoutafterruleprocessingKERNEL-thekerneluevent .032952]/devices/virtual/bdi/253:1UDEV.033720]/devices/virtual/bdi/253:1.033798]/devices/virtual/block/dm-1UDEV.033973]/devices/virtual/block/dm-1.034409]UDEV.047874]/devices/virtual/block/dm-1[root@node2[root@node2~]#udevadminfo-a-pUdevadminfostartswiththedevicespecifiedbythedevpathandwalksupthechainofparentdevices.Itprintsforeverydevicefound,allpossibleattributesintheudevruleskeyformat.Aruletomatch,canbecomposedbytheattributesoftheandtheattributesfromonesingleparent 0ATTRS{vendor}=="VMware,"ATTRS{model}=="VMwareVirtualS"ATTRS{rev}=="1.0" ##在node2上挂接上前一章配置的iscsiLUN挂接iscsi[root@node2~]#cat/etc/udev/rules.d/80-ACTION=="add",SUBSYSTEM=="block",DRIVERS=="sd",ENV{ID_MODEL}=="VIRTUAL-DISK",SYMLINK+="iscsi/NETDISK%n",MODE="0644"[root@node2SUBSYSTEM使用udevadminfoapsys/block/sdbDRIVERS同样使用udevadminfo-a-p/sys/block/sdbudevadminfo--export-db|grep-A10sdb|grep [root@node2~]#/etc/init.d/iscsirestart[root@node2~]#ll/dev/iscsi/totallrwxrwxrwx1rootroot6Aug1105:04NETDISK->lrwxrwxrwx1rootroot7Aug1105:04NETDISK1->../sdb1[root@node2~]#开发，将设备情况从/proc中分离出来并加以改进。 [root@ash6020[root@ash6020sys]#blockbusclassdevdevicesfirmwarefshypervisorkernelmodulepower[root@ash6020sys]# active-active模式，一方面可以做冗余，active-standby模式，只有一路链路工作，vendor-idRHCAInc.scsi_sns01initiator-addressinitiator-addressnode1/2/3/etc/init.d/iscsi重启node4tgtd[root@node4~]# -将#node2#node2做iSCSILUN[root@node2~]#iscsiadm-mdiscovery-tsendtargets-p [root@node2~]#iscsiadm-mdiscovery-tsendtargets-p #重启iscsi#[root@node2~]#ls/dev/disk/by-#--user_friendly_names[root@node2[root@node2~]#vim#default表示加入multipath的设备配置blacklist表示不加入multipath[root@node2~]#cat/etc/multipath.conf|egrep-v"^#|^$"defaults{ "/lib/udev/scsi_id--whiisted-- }}getuid_callout这行目的就是取设备的WWID #[root@node2~]#ls-l/dev/mapper/totallrwxrwxrwx1root 7Aug1202:471exampletarget->../dm-crw-rw1rootroot10,58Aug122013control[root@node2~]#user_friendly_names改成yes[root@node2~]#ls-l/dev/mapper/total0crw-rw1rootroot10,58Aug122013lrwxrwxrwx1root 7Aug1202:58mpatha->../dm-[root@node2~]#cat/etc/multipath.conf|egrep-v[root@node2~]#cat/etc/multipath.conf|egrep-v"^#|^$"defaults{getuid_callout"/lib/udev/scsi_id--whiisted-- }multipathsmultipathmultipath}}blacklist}mutipaths中wwid的取值用命令：/lib/udev/scsi_idwhiisted#此处不要加--re [root@node2~]#/etc/init.d/multipathdrestart[root@node2~]#ls-l/dev/mapper/total0crw-rw1rootroot10,58Aug122013lrwxrwxrwx1root 7Aug1203:02rhca-iscsidisk->../dm-#如果此处dm-0的指向设备名字未变，用multipathF#active状态，一条是enable[root@node2~]#multipath-size=1.0Gfeatures='0'hwhandler='0'wp=rw|- ='round-robin0'prio=1|`-6:0:0:1sdb8:16activeready`- ='round-robin0'prio=1`-5:0:0:1sdc8:32activeready[root@node2~]#[root@node2~]#llcrw-rw1rootroot10,58Aug122013lrwxrwxrwx1root 7Aug1203:05/dev/mapper/rhca-iscsidisk->../dm-brw-rw1rootdisk253,1Aug1203:13/dev/mapper/rhca-Aug1203:18:13|/lib/udev/scsi_idexittedwithAug1203:18:13|/lib/udev/scsi_idexittedwithsize=1.0Gfeatures='0'hwhandler='0'wp=undef ='round-robin0'prio=1|`-6:0:0:1sdb8:16activeready`- ='round-robin0'prio=1`-`-5:0:0:1sdc8:32activeready[root@node2~]#mkfs.ext4/dev/mapper/rhca-iscsidiskp1[root@node2~]#mkfs.ext4/dev/mapper/rhca-iscsidiskp1[root@node2~]#df-h/media/ SizeUsedAvailUse%Mounted 18M 2%#同时观察system的log，会发现检测到#同时观察system的log，会发现检测到iscsi一条链路断掉，在 #之后，切换到另外一条pathAug1203:28:08node2kernel:connection4:0:detectedconnerrorAug1203:28:09node2iscsid:KernelreportediSCSIconnection4:0error(1011)stateAug1203:28:11node2iscsid:cannotmakeaconnectionto:3260(-Aug1203:28:14node2iscsid:cannotmakeaconnectionto:3260(-Aug1203:28:17node2iscsid:cannotmakeaconnectionto:3260(-Aug1203:30:09node2kernel:device-mapper:multipath:FailingpathAug1203:30:09node2multipathd:rhca-iscsidisk:sdb-directiocheckerreportspathisdownAug1203:30:09node2multipathd:checkerfailedpath8:16inmaprhca-iscsidiskAug1203:30:09node2multipathd:rhca-iscsidisk:remainingactivepaths:#此时multipath状态显示，主链路faulty[root@node2media]#multipath-size=1.0Gfeatures='0'hwhandler='0'wp=rw ='round-robin0'prio=0|`-6:0:0:1sdb8:16failedfaulty`- ='round-robin0'prio=1`-5:0:0:1sdc8:32activeready#将eth2up，状态如下，activesize=1.0Gfeatures='0'hwhandler='0'wp=rw ='round-robin0'prio=1|`-6:0:0:1sdb8:16activeready`-`-='round-robin0'`-5:0:0:1sdc8:32activeready[root@node2media]#cat/etc/iscsi/iscsid.conf|greptimeout|[root@node2media]#cat/etc/iscsi/iscsid.conf|greptimeout|grep-v^# cement_timeout=120node.conn[0].timeo.login_timeout=15[root@node2media]#[root@node2media]#sed-i'/timeout/s/=.*$/=5/g'/etc/iscsi/iscsid.conf[root@node2media]#cat/etc/iscsi/iscsid.conf|greptimeout|grep-v^# cement_timeout=5[root@node2media]##RedHat只是PowerControl是使用的其他软件方式解决，而非一个硬件。RedHatHAclusterCorosyncModclusterdricci和luci之间通信的程CongaRedHatLuciRicciLucipythonweb所以Ricci在所有节点都需要，而Luci只需要装在一个节点上[root@node1~]#ssh-keygen-t[root@node1~]#ssh-keygen-t[root@node1~]#ssh-copy-id-i.ssh/id_rsa.pubroot@node2[root@node1~]#ssh-copy-id-i.ssh/id_rsa.pubroot@node3[root@node1~]#ssh-copy-id-i.ssh/id_rsa.pubroot@node4[root@node1/2/3/4~]#/etc/init.d/acpidstop[root@node1/2/3/4~]#chkconfigacpidoff[root@node4[root@node4~]#yum-ygroupinstall"HighAvailabilityManagement"[root@node4~]#/etc/init.d/lucistart#安装完 includepassword-auth includepassword-auth includepassword-auth includepassword-auth[root@node4~]#id-aluciuid=141(luci)gid=141(luci) #再退出用rootLuci[root@node1~]#yum-ygroupinstall'HighAvailability''ResilientStorage'[root@node2~]#yum-ygroupinstall'HighAvailability''ResilientStorage'[root@node3~]#yum-ygroupinstall'HighAvailability''ResilientStorage'#[root@node1/2/3~]#chkconfigriccion[root@node1/2/3~]#/etc/init.d/riccistart[root@node1/2/3~]#netstatnltp|grep:11111#设置ricci用户[root@node1/2/3~]#echoredhat|passwd--stdin登录ManageClusters=>ClusterNameUsethesamepasswordforallnodes:如果所有节点的ricci用户的一样，就可以勾选Node配置，这里使用的是private网段，node1/2.private.clu 是ricci用户的，同时注意11111端口是不是可以被 net到。然我更建议你在初期使用groupinstall方式。原因是Luci没有将集群的配置信息下放到那个节点。解决方式是，手动拷贝是Luci和下面节点的通信出现问题。出现问题查看系统日志/var/log/messagetailf这个问题是红帽预料到的，/etc/cluster/cluster.confnode2/3上。解决方式是手动拷贝过去。然后点击JionCluster^^值，则cluster正常，低于则cluster失效。Quorum=floor(“expectedvotes”/2+1Quorum票数是：每个节点拥有的票数的和/2Expectedvotes:3Expectedvotes:3 #Nodevotes: #Quorum: 3/21有5个节点可用，这样判定集群失效是有问题的，所以不要让单个node的票值过高。[root@node2~]#cat<?xml "nodeid="1" " "Expectedvotes:3Nodevotes:Quorum:第仲裁机 ClusterServer里叫做I/OFencing，他将仲裁磁盘和Fence功能结合在一起。node状态发生变化时候，通过节点间的竞争，看谁先抢到多数磁盘，抢到数量多的节点获得重组cluster权利，失败节点退出。KVMvmFencevirsh发送请求的方fencevmwareworkstationFencevmwareSCSI3PRiscsi-targetsFencelog里会报错：fence_scsi[error]failedtocreateRedHat手册里写道SCSIPersistentReservationsfence当使用SCSIfencing时，集群中的所有节点必须使用同一设备，这样每个节点即控制到每个LUN的，而不是对独立分区的。 Useoffence_scsiwithiSCSIstorageislimitedtoiSCSIserversvendortoensurethatyourserveriscompliantwithSCSI3ReservationReservationsupport.NotethattheiSCSIservershippedwithRedsoitisnotsuitableforusewithfence_scsi.Tips通过SCSIReservation机制来进行SCSI锁的操作，目前绝大多数的磁盘都支持SCSI到reservation reservation或者resettarget命令，用来解除SCSI锁。然后，第二个主机发送I/O请求之类型的SCSI锁。主机1上的HBA1对的LUN加上SCSI-2锁，此时即使主机1的HBA2也无法这个LUN。所以SCSI-2Reservation也被称为SinglePathReservation。PRKeyPRKeySCSI-3Reservation通常被应用在多通路的共享环境下面。这里SCSI-3Reservation也称之为PersistentReservation。所以在RHEL5时候fence_scsiclvmRHEL5的fence_scsi只支持SCSI2协以我把iscsi建立在ubuntu，配置方式请看附录中ubuntu下的iscsi配置##不 <?xml"votes="1""votes="1""votes="1" 是cluster.conf的语法错误。#显示success，配置成功了fencenode2success###[root@node2~]#yum-yinstall[root@node2~]#yum-yinstall[root@node2~]#chkconfigwatchdogon[root@desktop26~]#mkdir/etc/cluster/#生成key，考试时候这个key0.2544+0records4+0records4096bytes(4.1kB)copied, s,4.0#配置主机Fence，注意 [root@desktop26~]#fence_virtd-clibvirt0.1multicastListenermodulesareresponsibleforacceptingrequestsfromfencing ListenermoduleThemulticastlistenermoduleisdesignedforuseenvironmentswheretheguestsandhostsmaycommunicateoveranetworkusingThemulticastaddressistheaddressthata willusetosendfencingrequeststofence_virtd.MulticastIPAddressUsingipv4asfamily.onthatinterface.Normally,itlistensonallinterfaces.Inenvironmentswherethevirtualmachinesareusingthehostmachineasagateway,this*must*beset(typicallytovirbr0).Setto'none'fornointerface.Interface[none]:Thekeyfileisthesharedkeyinformationwhichisusedtoauthenticatefencingrequests.Thecontentsofthisfilemustbedistributedtoeachphysicalhostandvirtualmachinewithinacluster.KeyFileBackendmodulesareresponsibleforroutingrequeststotheappropriatehypervisorormanagementlayer.Thelibvirtbackendmoduleisdesignedforsingledesktopsorservers.Donotuseinenvironmentswherevirtualmachinesmaybemigratedbetweenhosts.Configurationcomplete.backends{libvirturi=}}

multicastport="1229";family="ipv4";address=key_file=}}backend="libvirt";listener=}===EndConfiguration ce/etc/fence_virt.confwiththeabove[y/N]?[root@desktop26~]#chkconfigfence_virtdon[root@node1~]#fence_nodenode2fencenode2Rgmanager资源组：由这些资源组成的一个groupUdevKERNEL=="sd*",--whiisted--recedevice=/dev/$name",",KERNEL=="sd*",--whiisted--rece-total0lrwxrwxrwx1rootroot6Aug1611:29hadisk->lrwxrwxrwx1rootroot7Aug1611:29hadisk1->lrwxrwxrwx1rootroot6Aug1611:29netdisk->../sdc[root@node1~]#[root@node1/2/3~]#yum-yinstall[root@node1~]#ipaddradddeveth00/24[root@node1~]#ipaddshowdeveth0link/ether00:0c:29:99:52:01brdff:ff:ff:ff:ff:ffinet1/24brd55scopeglobaleth0inet0/24scopeglobalsecondaryeth0valid_lftforeverpreferred_lftforever[root@node1~]#RHCA436这个启动吧，总会有点问题，不知道这个ricci[root@node2~]#ClusterStatusfortempcluster@SunAug1810:57:082013MemberStatus:QuorateMember 1 2Online,Local, 3Service Owner [root@node2~]#[root@node2~]#pkill-9httpd[root@node2~]#ps-ef|grephttp[root@node2~]#clustatClusterStatusfortempcluster@SunAug1810:59:382013MemberStatus:QuorateMember 1 2Online,Local, 3Service Owner [root@node2~]#clustatClusterStatusfortempcluster@SunAug1810:59:412013MemberStatus:QuorateMember 1 2Online,Local, 3Service Owner [root@node2~]#ClusterStatusfortempcluster@SunAug1811:00:032013MemberStatus:QuorateMember 1 2Online,Local, 3Service Owner [root@node2~]#[root@node2~]#ClusterStatusfortempcluster@SunAug1811:00:172013MemberStatus:QuorateMember 1 2Online,Local, 3Service Owner [root@node2~]#[root@node2~]#ClusterStatusfortempcluster@SunAug1811:03:142013MemberStatus:QuorateMember 1 2Online,Local, 3Online,Service Owner [root@node2~]#ClusterStatusfortempcluster@SunAug1811:03:512013MemberStatus:QuorateMember 1 2Online,Local, 3Online,Service Owner [root@node2~]#[root@node2~]#ClusterStatusfortempcluster@SunAug1811:04:062013MemberStatus:QuorateMember 1 2Online,Local, 3Online,Service Owner [root@node2~]#[root@node1~]#ClusterStatusfortempcluster@FriAug1612:32:422013MemberStatus:QuorateMember 1Online,Local, 2Online, 3Online,ServiceServiceOwnerCreatingbackingstorelogicalvolume...SUCCESSDefiningiSCSItarget...SUCCESSStartingSCSItargetdaemon:[root@node1~]#partprobe[OK#测试资源是否能在node1/2/3上加载，[root@node1~ipaddradddeveth10/24[root@node1~]#ipaddshoweth1[root@node1~]#servicehttpdStartinghttpd:Syntaxerroronline292ofRootmustbea#SELinux[root@node1~]#servicehttpdrestart [OK[root@node1~]#ls-Z[root@node1~]#elinks-dumpRHCA436 [OK[root@node1~]#umount/var/www/html/[root@node1~]#ipaddrdeldeveth10/24Node2/3 IP->filesystem->#资源的变化动态可以在#资源的变化动态可以在rgmanager的log#现有资源状态用clustat看[root@node1~]#ClusterStatusfortmpcluster@SatFeb2315:02:022013MemberStatus:QuorateMember ServiceOwnerClusterStatusfortempcluster@FriAug1612:36:462013MemberStatus:QuorateMember 1Online, 2 3Service Owner #[root@node1~]#clustatClusterStatusfortempcluster@FriAug1612:37:312013MemberStatus:QuorateMember 1Online,Local, 2Online, 3Online,Service Owner [root@node1~]#clusvcadm-ewebClusterStatusfortempcluster@FriAug1612:38:382013MemberStatus:QuorateMember 1Online,Local, 2Online, 3Online,Service Owner ##迁移资源组到别的node[root@node1~]#clusvcadm-rweb-m[root@node1~]#clusvcadm-d[root@node2~]#yum–yinstallsamba#分一个128M的分区用于测试#partporbe没反应，就重启下系统，这个是RHEL6的一个bug[root@node2~]#mkfs.ext4/dev/iscsi/hadisk2[root@node2cifs-export]#touchsamba-test[root@node2samba]#cat/etc/samba/smb.confworkgroup=logfile=/var/log/samba/log.%msecurity=userloadprinters=nocupsoptions=public=yes.D0FriAug1613:00:470MonAug1908:16:000FriAug1613:00:47D0FriAug1612:57:39 [root@node1~]#fdisk/dev/sda[root@node1~]#partprobe[root@node1~]#fdisk/dev/sda[root@node1~]#partprobe[root@node1~]#mkdir/nfs_data[root@node1~]#mount/dev/sda1/nfs_data/[root@node1~]#touch/nfs_data/nfs_test[root@node1~]#ipaddradddeveth10/24[root@node1~]#ipaddrshoweth1|grep172inet/24brd55scopeglobaleth1inet0/24scopeglobalsecondaryeth1[root@node1~]#umount[root@node1~]#ipaddrdeldeveth10/24[root@node1~]##node2/3<special<attributes<childtype="lvm"start="1"<childtype="fs"start="2"<childtype="clusterfs"start="3"<childtype="netfs"start="4"<child port"start="5"<child"start="6"<childtype="ip"start="7"<childtype="smb"start="8"<childtype="script"start="9"StartHA为了解决双节点的仲裁问题，RedHat解决方式是双节点设置预期仲裁票数为1，当节点数为2时候，cman会在配置文件中加入预期票数为1[root@node2~]#grep[root@node2~]#grepcman[root@node2~]#[root@node2~]#cman_toolcman_tool:nooperationspecifiedVersion:6.2.0ClusterId:32263ClusterMember:YesNodes:2Expectedvotes:Nodevotes:Quorum:Flags:2nodePortsBound:0NodeID:2Multicastaddresses:Nodeaddresses:LVMLVM#[root@node1~]#lvcreate-L2G-nresizemevgsrv[root@node1~]#mkdir/testresize[root@node1~]#touch/testresize/file{0..9}#模拟一个错误操作，在mount的情况下缩卷#[root@node1~]#lvcreate-L2G-nresizemevgsrv[root@node1~]#mkdir/testresize[root@node1~]#touch/testresize/file{0..9}#模拟一个错误操作，在mount的情况下缩卷Doyoureallywanttoreduceresizeme?[y/n]:yReducinglogicalvolumeresizemeto1.00GiBLogicalvolumeresizemesuccessfully#此时还是 [root@node1~]#umountmount:wrongfstype,badoption,badsuperblockon/dev/mapper/vgsrv-resizeme,missingcodepageorhelperprogram,orothererrorInsomecasesusefulinfoisfoundinsyslog-trydmesg|tailorso##查看LVM[root@node1~]#cat/etc/lvm/lvm.conf|greparchive|grep-v'#'archive=1[root@node1~]##connect()failedonlocalsocket:NosuchfileordirectoryInternalclusterlockinginitialisationfailed.WARNING:Fallingbacktolocalfile-basedVolumeGroupswiththeclusteredattributewillbeVGDescription:Created*before*executing'/sbin/vgs--noheadings-oBackupTime:SunNov1810:11:49VGDescription:Description:Created*before*executing'lvcreate-L2G-nresizemeBackupTime:TueFeb2612:17:12VGDescription:Created*before*executing'lvresizeL1Gdev/mapper/vgsrv-resizeme'#BackupTime:TueFeb2612:26:16VGDescription:Created*after*executing'lvresize-L1G/dev/mapper/vgsrv-BackupTime:TueFeb2612:26:16#这个其实就是卷的一个操作log。说明LVMblock[root@node1~]#file #[root@node1~]#vgcfgrestore-f.vg#此时仍然不能挂载的，将lv设置成noactive[root@node1~]lvchangeandev/mapper/vgsrv-resizeme[root@node1~]lvchangeay/dev/mapper/vgsrv-resizeme#remount完成blockblock状态，而不是拷贝一个新lvs[root@node4~]# OriginSnap%MoveLogCopy%vgsrv-wi-aovgsrv-wi- [root@node4~]##[root@node4~]#lvcreate-h|grep'\-[root@node4~]##为roo100M[root@node4~]#lvcreate-s/dev/vgsrv/root-nrootsnap-LRoundingupsizetofullphysicalextent128.00MiBLogicalvolume"rootsnap"created[root@node4~]#lvs OriginSnap%MoveLogCopy%Convert vgsrv-wi-ao256.00m vgsrvowi- rootsnapvgsrvswi-a-128.00m storagevgsrv-wi-ao #[root@node4~]#ls/net/ devhomelib64 mediamntopt rootselinuxsysusrbootetc lost+found netprocsbin tmp#测试，在根分区里dd#可以看到file这个文件并没有出现在/net里[root@node4~]#ll/file-rw-r--r--.1rootroot Feb2614:21/file[root@node4~]#ll/net/filels:cannotaccess/net/file:Nosuchfileor#[root@node4~]# OriginSnap%MoveLogCopy%Convert vgsrv-wi-ao256.00m vgsrvowi- rootsnapvgsrvswi-ao128.00m storagevgsrv-wi-ao [root@node4~]# OriginSnap%MoveLogCopy%Convert vgsrv-wi-ao256.00m vgsrvowi- rootsnapvgsrvswi-ao128.00m storagevgsrv-wi-ao #将快照恢复到root##Can'tmergeoveropenoriginvolumeCan'tmergewhensnapshotisMergingofsnapshotrootsnapwillstartnextactivation.[root@node4~]#reboot#快照合并之后，逻辑卷要重新激活，使用lvchange–anay#根分区要重新激活，这个只能重启了#[root@node4~]# OriginSnap%MoveLogCopy%vgsrv-wi-aovgsrv-wi- [root@node4~]#HA-HA-LVMnode[root@node1~]#/etc/init.d/clvmdstartStartingclvmd:ActivatingVG(s): 4logicalvolume(s)involumegroup"vgsrv"nowactiveclvmdnotrunningonnodenode3.private.cluclvmdnotrunningonnode[OK[root@node1~]#chkconfigclvmd-- [root@node1~]#chkconfigclvmdonnode2/3#设置分区为8e，注意使用-cu参数node1/2/3partporbe和[root@node1~]#pvcreate/dev/mapper/clusterstoragep5[root@node1[root@node1~]#lvcreate-nhttplv-L128M[root@node2~]# OriginSnap%MoveLogCopy%Converthttplvfirstvg-wi-a-128.00m-wi-ao-wi-ao[root@node1~]#echo"ClusterLVMTestforRHCA436">/net/index.html[root@node1~]#chcon-R-thttpd_sys_content_t/net[root@node1~]#umount[root@node1~]##PV#LV#SN110wz--nc1.01g140wz--n-#[root@node1~]#lvchange-an[root@node1~]# OriginSnap%MoveLogCopy%Convert firstvg-wi128.00m resizeme-wi-ao [root@node1~]#[root@node1~]#clusvcadm-d##<fsdevice="/dev/firstvg/httplv"fsid="4657"fstype="ext4"mountpoint="/var/www/html/"name="halvm_web"quick_status="on"self_fence="on"/>[root@node1~]#clusvcadm-e[root@node1~]#ClusterStatusfortmpcluster@ThuFeb2803:55:292013MemberStatus:QuorateMember Online,Local,ServiceOwner#配置 #配置 #将测试环境清空还原，lab-*是教学环境所有，考试中没有这个的[root@node4~]#lab-setup-targets[root@node1/2/3~]#lab-setup-iscsi-#[root@node1/2/3~]#yum-ygroupinstall'HighAvailability''ResilientStorage'[root@node4~]#yumgroupinstall'HighAvailabilityManagement'#node1/2/3设置Ricci[root@node1~]#echoredhat|passwd--stdinricciChangingpasswordforuserricci.[root@node1~]#/etc/init.d/riccistartnode4[root@node4~]#chkconfiglucion[root@node1/2/3~]#chkconfigcmanon所有节点，尽可能少用Luci在哪个节点上 ##安装libvirt-fence[root@desktop26~]#fence_virtd-libvirt0.1multicastfrom ListenermoduleThemulticastlistenermoduleisdesignedforusewheretheguestsandhostsmaycommunicateoveranetworkusingThemulticastaddressistheaddressthatsendfencingrequeststowilluseUsingipv4asMulticastIPPortonthatinterface.Normally,itlistensonallinterfaces.Inenvironmentswherethevirtualmachinesareusingthehostmachineasagateway,this*must*beset(typicallytovirbr0).Setto'none'fornointerface.Interface[none]:Thekeyfileisthesharedkeyinformationwhichisusedtoauthenticatefencingrequests.Thecontentsofthisfilemustbedistributedtoeachphysicalhostandvirtualmachinewithinacluster.KeyFileBackendmodulesareresponsibleforroutingrequeststotheappropriatehypervisorormanagementlayer.Backendmodule Nobackendmodulenamedlibvirtfound!（#为什么出现这个错误？因为我写libvirtConfigurationcomplete.backends{libvirturi=}}

multicastport="1229";family="ipv4";address=key_file=}}backend="libvirt";listener=}===EndConfiguration [root@desktop26~]##上面配置中有个fencekey[root@desktop26~]#scp/etc/cluster/fence_xvm.keynode1:/etc/cluster/[root@desktop26~]#scp/etc/cluster/fence_xvm.keynode2:/etc/cluster/[root@desktop26~]#scp/etc/cluster/fence_xvm.keynode3:/etc/cluster/[root@desktop26~]#/etc/init.d/fence_virtdstart#node1fencenode1successRestricted:##分一个2G的分区，并设置成8e，过程略#三个节点发现0[root@node1/2/3~]#partprobe#三个节点启动clvmd，并打开lvmcluster模式[root@node1/2/3~]#/etc/init.d/clvmdstart[root@node1/2/3~]#chkconfigclvmdon[root@node1/2/3~]#lvmconf--enable-cluster[root@node2~]#pvcreate/dev/mapper/clusterstoragep1[root@node2~]#lvcreate-nnfslv-L1Gnfsvg#只要clvmd是启动，在任何节点，nfslv都是正常的激活状态，nfsvg显示cluster[root@node2~]#vgs|grepnfsvg [root@node2~]#lvs|grepnfslvnfslvnfsvg-wi- nfslvnfsvg-wi-a- #格式化gfs2mkfs.gfs2[options]<device>[block-count-b-c--EnabledebuggingcodePrintthishelp,then-J Sizeof -j Numberof - Don'ttrytodiscardunused- Don'taskfor-p - Don'tprint-r ResourceGroup#<clustername>:<table-u Sizeofunlinked- [root@node2~]#mkfs.gfs2-tcluster1:locktable1-j3-J128M/dev/mapper/nfsvg-#[root@node2~]#df-T/dev/mapper/nfsvg-nfslv UsedAvailableUse%Mounted65125238%-t NameofthelockNFSServer 二个service）#[root@node2~]#ClusterStatusforcluster1@SunMar315:49:312013MemberStatus:QuorateMember 1Online, 2Online,Local, 3Online,Service Owner [root@node2~]#[root@node2~]#clusvcadm-e#Failure总有原因，查看日志[root@node2~]#tail-100fMar0315:52:08rgmanager[nfsserver]StartingNFSServernfs_expMar0315:52:09rgmanager[nfsserver]Startingrpc.statdMar0315:52:09rgmanager[nfsserver]StartingNFSMar0315:52:09rgmanager[nfsserver]FailedtostartNFSMar0315:52:09rgmanager[nfsserver]FailedtostartNFSServerMar0315:52:09rgmanagerstartonnfsserver"nfs_exp"returned1(genericerror)Mar0315:52:09rgmanager#68:Failedtostartservice:nfs_service;returnvalue:1Mar0315:52:09rgmanagerStopserviceservice:nfs_serviceShuttingdownNFS ShuttingdownNFS ShuttingdownNFS StartingNFS [OKStartingNFS [OKStartingNFS # [OKStarting [OKShuttingdownNFSShuttingdownNFS]ShuttingdownNFS[OKShuttingdownNFS[OKStartingNFS[OKStartingNFS[OKStartingNFS[]StartingNFS[]Localmachinetryingtoenableservice:nfs_ser