Moosefs 分布式存储

1、=Moosefs 分布式存储=rootserver41 # lftp i #远程访问服务器，获取mfs安装包lftp i:> cd pub/docs/mfs/错误：显示是软件包的名称中有的-1不识别，需要修改软件包名称error: Failed build dependencies:错误：需要安装fuse-devel依赖性rootserver41 # yum install -y fuse-develrootserver41 # cd /root/rpmbuild/RPMS/x86_64/rootserver41 x86_64# cd /etc/mfs/rootserver41 mfs#

2、 cd /var/lib/mfs/rootserver41 mfs# chown nobody /var/lib/mfs/ -Rrootserver41 mfs# mfsmaster rootserver41 mfs# cd /var/lib/mfs/rootserver41 mfs# lsrootserver41 mfs# mfsmaster stoprootserver41 mfs# lsrootserver41 mfs# mfsmasterrootserver41 mfs# lsrootserver41 mfs# cd /usr/share/mfscgi/rootserver41 mfs

3、cgi# chmod +x chart.cgi mfs.cgi rootserver41 mfscgi# mfscgiserv rootserver41 # cd rpmbuild/RPMS/x86_64/rootserver42 mfs# vim /etc/hostsrootserver42 mfs# fdisk -lrootserver42 mfs# fdisk -cu /dev/vdcCommand (m for help: nCommand actione extendedp primary partition (1-4pPartition number (1-4: 1First se

4、ctor (2048-16777215, default 2048: Using default value 2048Last sector, +sectors or +sizeK,M,G (2048-16777215, default 16777215: Using default value 16777215Command (m for help: tSelected partition 1Hex code (type L to list codes: 8eChanged system type of partition 1 to 8e (Linux LVMCommand (m for h

5、elp: pDisk /dev/vdc: 8589 MB, 8589934592 bytes2 heads, 1 sectors/track, 8388608 cylinders, total 16777216 sectorsUnits = sectors of 1 * 512 = 512 bytesSector size (logical/physical: 512 bytes / 512 bytesI/O size (minimum/optimal: 512 bytes / 512 bytesDisk identifier: 0xb2f86748Device Boot Start End

6、Blocks Id System/dev/vdc1 1025 8388608 8387584 8e Linux LVMCommand (m for help: wThe partition table has been altered!Calling ioctl( to re-read partition table.Syncing disks.rootserver42 mfs# pvcreate /dev/vdc1rootserver42 mfs# vgcreate mfsvg /dev/vdc1 rootserver42 mfs# lvcreate -L 4g -n demo mfsvgr

7、ootserver42 mfs# mkfs.ext4 /dev/mfsvg/demo rootserver42 mfs# mkdir /mnt/chunk1rootserver42 mfs# mount /dev/mfsvg/demo /mnt/chunk1/rootserver42 mfs# vim /etc/fstab /dev/mfsvg/demo /mnt/chunk1 ext4 defaults 0 0rootserver42 mfs# mount -arootserver42 mfs# df -hFilesystem Size Used Avail Use% Mounted on/

8、dev/mapper/VolGroup-lv_root 4.9G 1.1G 3.6G 23% /tmpfs 246M 0 246M 0% /dev/shm/dev/vda1 485M 33M 427M 8% /boot/dev/mapper/mfsvg-demo 4.0G 136M 3.7G 4% /mnt/chunk1rootserver42 mfs# vim mfshdd.cfg/mnt/chunk1rootserver42 mfs# chown -R nobody.nobody /mnt/chunk1/rootserver42 mfs# mkdir /var/lib/mfsrootser

9、ver42 mfs# chown -R nobody /var/lib/mfs/rootserver42 mfs# mfschunkserverrootserver42 mfs# cd /var/lib/mfs/rootserver42 mfs# l. . .mfschunkserver.lockrootserver42 # lvextend -l +1023 /dev/mfsvg/demo rootserver42 # resize2fs /dev/mfsvg/demo rootserver42 # df -hFilesystem Size Used Avail Use% Mounted o

10、n/dev/mapper/VolGroup-lv_root 4.9G 1.1G 3.6G 23% /tmpfs 246M 0 246M 0% /dev/shm/dev/vda1 485M 33M 427M 8% /boot/dev/mapper/mfsvg-demo 7.9G 138M 7.4G 2% /mnt/chunk1rootserver43 # vim /etc/hostsrootserver43 # fdisk -cu /dev/vdbCommand (m for help: pDisk /dev/vdb: 8589 MB, 8589934592 bytes2 heads, 1 se

11、ctors/track, 8388608 cylinders, total 16777216 sectorsUnits = sectors of 1 * 512 = 512 bytesSector size (logical/physical: 512 bytes / 512 bytesI/O size (minimum/optimal: 512 bytes / 512 bytesDisk identifier: 0x28340149Device Boot Start End Blocks Id System/dev/vdb1 2048 16777215 8387584 8e Linux LV

12、MCommand (m for help: dSelected partition 1Command (m for help: pDisk /dev/vdb: 8589 MB, 8589934592 bytes2 heads, 1 sectors/track, 8388608 cylinders, total 16777216 sectorsUnits = sectors of 1 * 512 = 512 bytesSector size (logical/physical: 512 bytes / 512 bytesI/O size (minimum/optimal: 512 bytes /

13、 512 bytesDisk identifier: 0x28340149Device Boot Start End Blocks Id SystemCommand (m for help: nCommand actione extendedp primary partition (1-4pPartition number (1-4: 1First sector (2048-16777215, default 2048: Using default value 2048Last sector, +sectors or +sizeK,M,G (2048-16777215, default 167

14、77215: Using default value 16777215Command (m for help: tSelected partition 1Hex code (type L to list codes: 8eChanged system type of partition 1 to 8e (Linux LVMCommand (m for help: pDisk /dev/vdb: 8589 MB, 8589934592 bytes2 heads, 1 sectors/track, 8388608 cylinders, total 16777216 sectorsUnits = s

15、ectors of 1 * 512 = 512 bytesSector size (logical/physical: 512 bytes / 512 bytesI/O size (minimum/optimal: 512 bytes / 512 bytesDisk identifier: 0x28340149Device Boot Start End Blocks Id System/dev/vdb1 2048 16777215 8387584 8e Linux LVMCommand (m for help: wThe partition table has been altered!Cal

16、ling ioctl( to re-read partition table.rootserver43 # pvcreate /dev/vdb1 rootserver43 # vgcreate mfsvg /dev/vdb1 rootserver43 # lvcreate -l 2047 -n demo mfsvgrootserver43 # mkfs.ext4 /dev/mfsvg/demo rootserver43 # mkdir /mnt/chunk1rootserver43 mfs# vim mfshdd.cfg/mnt/chunk1rootserver43 mfs# chown -R

17、 nobody.nobody /mnt/chunk1/rootserver43 mfs# mkdir /var/lib/mfsrootserver43 mfs# chown -R nobody /var/lib/mfs/rootserver43 mfs# mfschunkserverrootserver43 mfs# cd /var/lib/mfs/rootserver43 mfs# l. . .mfschunkserver.lockerror: Failed dependencies:rootserver44 # cd /etc/mfs/rootserver44 mfs# vim /etc/

18、hostsrootserver44 mfs# vim mfsmount.cfg/mnt/mfsrootserver44 mfs# mkdir /mnt/mfsrootserver44 mfs# mfsmount mfsmaster accepted connection with parameters: read-write,restricted_ip ; root mapped to root:rootrootserver44 mfs# cp /etc/fstab .rootserver44 mfs# mfsfileinfo fstab fstab:chunk 0: 000000000000

19、0001_00000001 / (id:1 ver:1rootserver44 mfs# mkdir dir1 dir2rootserver44 mfs# mkdir dir1 dir2rootserver44 mfs# mv fstab dir1/rootserver44 mfs# mfssetgoal -r 2 dir2/dir2/:inodes with goal changed: 1inodes with goal not changed: 0inodes with permission denied: 0rootserver43 mfs# mfschunkserver stoproo

20、tserver44 dir2# mfsfileinfo fstabfstab:chunk 0: 0000000000000002_00000001 / (id:2 ver:1rootserver42 chunk1# mfschunkserver stoprootserver44 dir2# mfsfileinfo fstabfstab:chunk 0: 0000000000000002_00000001 / (id:2 ver:1no valid copies !rootserver42 chunk1# mfschunkserver rootserver44 dir2# mfsfileinfo

21、 fstabfstab:chunk 0: 0000000000000002_00000001 / (id:2 ver:1rootserver44 dir2# vim fstab 修改fstab文件中的内容sysfs /sys sysfs defaults 0 0proc /proc proc defaults 0 0proc /proc proc defaults 0 0proc /proc proc defaults 0 0rootserver43 mfs# mfschunkserver 修改文件后，打开server43主机的mfschunkserver服务，文件不能达到同步，所以不会显示s

22、erver43主机rootserver44 dir2# mfsfileinfo fstabfstab:chunk 0: 0000000000000004_00000001 / (id:4 ver:1rootserver44 dir2# dd if=/dev/zero of=bigfile bs=1M count=200建立一个200M的bigfile文件rootserver44 dir2# mfsfileinfo bigfile bigfile:chunk 0: 0000000000000007_00000001 / (id:7 ver:1chunk 1: 0000000000000008_0

23、0000001 / (id:8 ver:1chunk 2: 0000000000000009_00000001 / (id:9 ver:1chunk 3: 000000000000000A_00000001 / (id:10 ver:1两个server同时写入，为了增加读写性能；显示bigfile文件，说明200M的文件被分为4个块，每个块为50M。rootserver44 dir2# rm -f fstab rootserver44 dir2# mkdir /mnt/metarootserver44 dir2# mfsmount -m /mnt/meta/ -H mfsmastermfsma

24、ster accepted connection with parameters: read-write,restricted_iprootserver44 dir2# cd /mnt/meta/trash/rootserver44 trash# ls00000005|dir2|fstab 0000000A|dir2|fstab 0000000F|dir2|.fstab.swpx00000006|dir2|.fstab.swp 0000000B|dir2|.fstab.swp 00000010|dir2|.fstab.swp00000007|dir2|.fstab.swpx 0000000C|

25、dir2|.fstab.swpx undel00000008|dir2|.fstab.swp 0000000D|dir2|.fstab.swp00000009|dir2|4913 0000000E|dir2|.fstab.swprootserver44 trash# mv 0000000A|dir2|fstab undel/rootserver44 trash# cd /mnt/mfs/dir2/rootserver44 dir2# lsbigfile fstabrootserver44 dir2# mfsgettrashtime /mnt/mfs/dir2/mnt/mfs/dir2/: 86

26、400显示/mnt/mfs/dir2/目录的保留时间为86400s=24h，即用户误删文件后，在24h内可以用上述方法进行恢复。rootserver41 # ps ax1262 ? S< 0:09 mfsmasterrootserver41 # kill -9 1262rootserver41 # cd /var/lib/mfs/rootserver41 mfs# lltotal 784-rw-r- 1 nobody nobody 548 Aug 4 16:00 sessions.mfs-rw-r- 1 nobody nobody 762516 Aug 4 16:00 stats.mfs

27、rootserver41 mfs# mfsmaster init: file system manager failed !error occured during initialization - exiting此时mfsmaster服务启动不了rootserver41 mfs# mfsmetarestore -a重新载入rootserver41 mfs# mfsmaster 重新载入后重启mfsmaster服务，能正常启动rootserver41 mfs# cd /root/rpmbuild/RPMS/x86_64/rootserver42 # cd /etc/mfs/rootserver

28、42 mfs# chown -r nobody /var/lib/mfs/rootserver44 # umount -l /mnt/mfs/ #客户端卸载 MooseFS 文件系统rootserver42 mfs# mfschunkserver stoprootserver43 mfs# mfschunkserver stoprootserver41 # /etc/init.d/drbd startrootserver42 # /etc/init.d/drbd startrootserver41 # drbdadm primary examplerootserver41 # cat /pro

29、c/drbd 1: cs:Connected ro:Primary/Secondary ds:UpToDate/Diskless C r-ns:0 nr:0 dw:0 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0rootserver41 # mount /dev/drbd1 /mnt/rootserver41 # cd /var/lib/mfs/rootserver41 mfs# mv * /mnt/rootserver41 # chown nobody /mnt/rootserver41 # umount /mnt/rootserv

30、er41 ha.d# cd /etc/init.d/rootserver41 init.d# vim mfs#!/bin/bash# Init file for the MooseFS master service# chkconfig: - 92 84# description: MooseFS master# processname: mfsmaster# Source function library.# Source networking configuration. /etc/init.d/functions. /etc/sysconfig/network# Source initi

31、alization configuration.# Check that networking is up. "$NETWORKING" = "no" && exit 0 -x "/usr/sbin/mfsmaster" | exit 1 -r "/etc/mfs/mfsmaster.cfg" | exit 1 -r "/etc/mfs/mfsexports.cfg" | exit 1RETVAL=0prog="mfsmaster"datadir="

32、/var/lib/mfs"mfsbin="/usr/sbin/mfsmaster"mfsrestore="/usr/sbin/mfsmetarestore"start ( echo -n $"Starting $prog: "$mfsbin start >/dev/null 2>&1if $? -ne 0 ;then$mfsrestore -a >/dev/null 2>&1 && $mfsbin start >/dev/null 2>&1fiRET

33、VAL=$?echoreturn $RETVALstop ( echo -n $"Stopping $prog: "$mfsbin -s >/dev/null 2>&1 | killall -9 $prog #>/dev/null 2>&1RETVAL=$?echoreturn $RETVALrestart ( stopstartreload ( echo -n $"reload $prog: "$mfsbin reload >/dev/null 2>&1RETVAL=$?echoreturn

34、$RETVALrestore ( echo -n $"restore $prog: "$mfsrestore -a >/dev/null 2>&1RETVAL=$?echoreturn $RETVALcase "$1" instartstart;stopstop;restartrestart;reloadreload;restorerestore;statusstatus $progRETVAL=$?;*echo $"Usage: $0 start|stop|restart|reload|restore|status"RETVAL=1esacexit $RETVALrootserver41 init.d# chmod +x mfsrootserver41 # mount /dev/drbd1 /var/lib/mfs/rootserver41 # /etc/init.d/mfs startrootserver41 # vim /etc/ha.d/haresourcesrootserver42 # vim /etc/hostsrootserver43 # vim /etc/hostsMfsmasterrootserver44 # vim