Hadoop实验二-熟悉常用的HDFS操作

Hadoop实验⼆——熟悉常⽤的HDFS操作1.编程实现以下指定功能，并利⽤Hadoop提供的Shell命令完成相同任务：（1）向HDFS中上传任意⽂本⽂件，如果指定的⽂件在HDFS中已经存在，由⽤户指定是追加到原有⽂件末尾还是覆盖原有的⽂件；shell命令实现⾸先启动所有的hadoop应⽤上传本地⽂件到HDFShadoopfs-puttext.txt/Test/追加到⽂件末尾的指令hadoopfs-appendToFilelocal.txt/Test/text.txt查看HDFS⽂件的内容hadoopfs-cat/Test/text.txt覆盖原有⽂件的指令hadoopfs-copyFromLocal-flocal.txt/Test/text.txt以上过程也可以⽤shell脚本实现：if$(hadoopfs-test-etext.txt);then$(hadoopfs-appendToFilelocal.txt/Test/text.txt);else$(hadoopfs-copyFromLocal-flocal.txt/Test/text.txt);fiJAVA实现importjava.io.FileInputStream;importjava.io.IOException;importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.FSDataOutputStream;importorg.apache.hadoop.fs.FileSystem;importorg.apache.hadoop.fs.Path;publicclassCopyFromLocalFile{/***判断路径是否存在*/publicstaticbooleantest(Configurationconf,Stringpath){try(FileSystemfs=FileSystem.get(conf)){returnfs.exists(newPath(path));}catch(IOExceptione){e.printStackTrace();returnfalse;}}/***复制⽂件到指定路径若路径已存在，则进⾏覆盖*/publicstaticvoidcopyFromLocalFile(Configurationconf,StringlocalFilePath,StringremoteFilePath){PathlocalPath=newPath(localFilePath);PathremotePath=newPath(remoteFilePath);try(FileSystemfs=FileSystem.get(conf)){/*fs.copyFromLocalFile第⼀个参数表⽰是否删除源⽂件，第⼆个参数表⽰是否覆盖*/fs.copyFromLocalFile(false,true,localPath,remotePath);}catch(IOExceptione){e.printStackTrace();}}/***追加⽂件内容*/publicstaticvoidappendToFile(Configurationconf,StringlocalFilePath,StringremoteFilePath){PathremotePath=newPath(remoteFilePath);try(FileSystemfs=FileSystem.get(conf);FileInputStreamin=newFileInputStream(localFilePath);){FSDataOutputStreamout=fs.append(remotePath);byte[]data=newbyte[1024];intread=-1;while((read=in.read(data))>0){out.write(data,0,read);}out.close();}catch(IOExceptione){e.printStackTrace();}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringlocalFilePath="/usr/local/hadoop/text.txt";//本地路径StringlocalFilePath="/usr/local/hadoop/text.txt";//本地路径StringremoteFilePath="/user/tiny/text.txt";//HDFS路径//Stringchoice="append";//若⽂件存在则追加到⽂件末尾Stringchoice="overwrite";//若⽂件存在则覆盖try{/*判断⽂件是否存在*/booleanfileExists=false;if(CopyFromLocalFile.test(conf,remoteFilePath)){fileExists=true;System.out.println(remoteFilePath+"已存在.");}else{System.out.println(remoteFilePath+"不存在.");}/*进⾏处理*/if(!fileExists){//⽂件不存在，则上传CopyFromLocalFile.copyFromLocalFile(conf,localFilePath,remoteFilePath);System.out.println(localFilePath+"已上传⾄"+remoteFilePath);}elseif(choice.equals("overwrite")){//选择覆盖CopyFromLocalFile.copyFromLocalFile(conf,localFilePath,remoteFilePath);System.out.println(localFilePath+"已覆盖"+remoteFilePath);}elseif(choice.equals("append")){//选择追加CopyFromLocalFile.appendToFile(conf,localFilePath,remoteFilePath);System.out.println(localFilePath+"已追加⾄"+remoteFilePath);}}catch(Exceptione){e.printStackTrace();}}}（2）从HDFS中下载指定⽂件，如果本地⽂件与要下载的⽂件名称相同，则⾃动对下载的⽂件重命名；Shell命令实现if$(hadoopfs-test-e/usr/local/hadoop/text.txt);then$(hadoopfs-copyToLocal/Test/text.txt./text.txt);else$(hadoopfs-copyToLocal/Test/text.txt./text2.txt);fiJAVA实现：importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;importorg.apache.hadoop.fs.FileSystem;importjava.io.*;publicclassCopyToLocal{/***下载⽂件到本地判断本地路径是否已存在，若已存在，则⾃动进⾏重命名*/publicstaticvoidcopyToLocal(Configurationconf,StringremoteFilePath,StringlocalFilePath){PathremotePath=newPath(remoteFilePath);try(FileSystemfs=FileSystem.get(conf)){Filef=newFile(localFilePath);/*如果⽂件名存在，⾃动重命名(在⽂件名后⾯加上_0,_1...)*/if(f.exists()){System.out.println(localFilePath+"已存在.");Integeri=Integer.valueOf(0);while(true){f=newFile(localFilePath+"_"+i.toString());if(!f.exists()){localFilePath=localFilePath+"_"+i.toString();break;}else{i++;continue;}}System.out.println("将重新命名为:"+localFilePath);}//下载⽂件到本地PathlocalPath=newPath(localFilePath);fs.copyToLocalFile(remotePath,localPath);}catch(IOExceptione){//TODOAuto-generatedcatchblocke.printStackTrace();}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringlocalFilePath="/usr/local/hadoop/text.txt";//本地路径StringremoteFilePath="/user/tiny/text.txt";//HDFS路径try{CopyToLocal.copyToLocal(conf,remoteFilePath,localFilePath);System.out.println("下载完成");}catch(Exceptione){e.printStackTrace();}}}（3）将HDFS中指定⽂件的内容输出到终端中；Shell脚本实现：hadoopfs-cat/Test/text.txtJAVA实现importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;importorg.apache.hadoop.fs.FileSystem;importjava.io.*;publicclassCat{/***读取⽂件内容*/publicstaticvoidcat(Configurationconf,StringremoteFilePath){PathremotePath=newPath(remoteFilePath);try(FileSystemfs=FileSystem.get(conf);FSDataInputStreamin=fs.open(remotePath);BufferedReaderd=newBufferedReader(newInputStreamReader(in));){Stringline;while((line=d.readLine())!=null){System.out.println(line);}}catch(IOExceptione){e.printStackTrace();}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringremoteFilePath="/user/tiny/input/text.txt";//HDFS路径try{System.out.println("读取⽂件:"+remoteFilePath);Cat.cat(conf,remoteFilePath);System.out.println("\n读取完成");}catch(Exceptione){e.printStackTrace();}}}（4）显⽰HDFS中指定的⽂件的读写权限、⼤⼩、创建时间、路径等信息；Shell命令实现hadoopfs-ls-h/Test/text.txtJAVA实现：importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;importorg.apache.hadoop.fs.FileSystem;importjava.io.*;importjava.text.SimpleDateFormat;publicclassList{/***显⽰指定⽂件的信息*/publicstaticvoidls(Configurationconf,StringremoteFilePath){try(FileSystemfs=FileSystem.get(conf)){PathremotePath=newPath(remoteFilePath);FileStatus[]fileStatuses=fs.listStatus(remotePath);for(FileStatuss:fileStatuses){System.out.println("路径:"+s.getPath().toString());System.out.println("权限:"+s.getPermission().toString());System.out.println("⼤⼩:"+s.getLen());/*返回的是时间戳,转化为时间⽇期格式*/longtimeStamp=s.getModificationTime();SimpleDateFormatformat=newSimpleDateFormat("yyyy-MM-ddHH:mm:ss");Stringdate=format.format(timeStamp);System.out.println("时间:"+date);}}catch(IOExceptione){e.printStackTrace();}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringremoteFilePath="/user/tiny/text.txt";//HDFS路径try{System.out.println("读取⽂件信息:"+remoteFilePath);List.ls(conf,remoteFilePath);System.out.println("\n读取完成");}catch(Exceptione){e.printStackTrace();}}}（5）给定HDFS中某⼀个⽬录，输出该⽬录下的所有⽂件的读写权限、⼤⼩、创建时间、路径等信息，如果该⽂件是⽬录，则递归输出该⽬录下所有⽂件相关信息；Shell命令实现：hadoopfs-ls-R-h/TestJAVA实现：importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;importorg.apache.hadoop.fs.FileSystem;importjava.io.*;importjava.text.SimpleDateFormat;publicclassListDir{/***显⽰指定⽂件夹下所有⽂件的信息（递归）*/publicstaticvoidlsDir(Configurationconf,StringremoteDir){try(FileSystemfs=FileSystem.get(conf)){PathdirPath=newPath(remoteDir);/*递归获取⽬录下的所有⽂件*/RemoteIterator<LocatedFileStatus>remoteIterator=fs.listFiles(dirPath,true);/*输出每个⽂件的信息*/while(remoteIterator.hasNext()){FileStatuss=remoteIterator.next();System.out.println("路径:"+s.getPath().toString());System.out.println("权限:"+s.getPermission().toString());System.out.println("⼤⼩:"+s.getLen());/*返回的是时间戳,转化为时间⽇期格式*/LongtimeStamp=s.getModificationTime();SimpleDateFormatformat=newSimpleDateFormat("yyyy-MM-ddHH:mm:ss");Stringdate=format.format(timeStamp);System.out.println("时间:"+date);System.out.println();}}catch(IOExceptione){e.printStackTrace();}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringremoteDir="/user/tiny";//HDFS路径try{System.out.println("(递归)读取⽬录下所有⽂件的信息:"+remoteDir);ListDir.lsDir(conf,remoteDir);System.out.println("读取完成");}catch(Exceptione){e.printStackTrace();}}}（6）提供⼀个HDFS内的⽂件的路径，对该⽂件进⾏创建和删除操作。如果⽂件所在⽬录不存在，则⾃动创建⽬录；Shell脚本实现：#!/bin/bashif$(hadoopfs-test-d/Test/test1);then$(hadoopfs-touchz/Test/test1/text1.txt);else$(hadoopfs-mkdir-p/Test/test1&&hdfsdfs-touchz/Test/test1/text1.txt);fiJAVA实现：importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;importjava.io.*;publicclassRemoveOrMake{/***判断路径是否存在*/publicstaticbooleantest(Configurationconf,Stringpath){try(FileSystemfs=FileSystem.get(conf)){returnfs.exists(newPath(path));}catch(IOExceptione){e.printStackTrace();returnfalse;}}/***创建⽬录*/publicstaticbooleanmkdir(Configurationconf,StringremoteDir){try(FileSystemfs=FileSystem.get(conf)){PathdirPath=newPath(remoteDir);returnfs.mkdirs(dirPath);}catch(IOExceptione){e.printStackTrace();returnfalse;}}/***创建⽂件*/publicstaticvoidtouchz(Configurationconf,StringremoteFilePath){PathremotePath=newPath(remoteFilePath);try(FileSystemfs=FileSystem.get(conf)){FSDataOutputStreamoutputStream=fs.create(remotePath);outputStream.close();}catch(IOExceptione){e.printStackTrace();}}/***删除⽂件*/publicstaticbooleanrm(Configurationconf,StringremoteFilePath){PathremotePath=newPath(remoteFilePath);try(FileSystemfs=FileSystem.get(conf)){returnfs.delete(remotePath,false);}catch(IOExceptione){e.printStackTrace();e.printStackTrace();returnfalse;}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringremoteFilePath="/user/tiny/input/text.txt";//HDFS路径StringremoteDir="/user/tiny/input";//HDFS路径对应的⽬录try{/*判断路径是否存在，存在则删除，否则进⾏创建*/if(RemoveOrMake.test(conf,remoteFilePath)){RemoveOrMake.rm(conf,remoteFilePath);//删除System.out.println("删除⽂件:"+remoteFilePath);}else{if(!RemoveOrMake.test(conf,remoteDir)){//若⽬录不存在，则进⾏创建RemoveOrMake.mkdir(conf,remoteDir);System.out.println("创建⽂件夹:"+remoteDir);}RemoveOrMake.touchz(conf,remoteFilePath);System.out.println("创建⽂件:"+remoteFilePath);}}catch(Exceptione){e.printStackTrace();}}}（8）向HDFS中指定的⽂件追加内容，由⽤户指定内容追加到原有⽂件的开头或结尾；Shell命令实现：追加到⽂件结尾hadoopfs-appendToFilelocal.txt/Test/text.txt追加到⽂件开头hadoopfs-get/Test/text.txtcat/Test/text.txt>>local.txthadoopfs-copyFromLocal-ftext.txt/Test/text.txtJAVA实现：importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;importorg.apache.hadoop.fs.FileSystem;importjava.io.*;publicclassAppendToFile{/***判断路径是否存在*/publicstaticbooleantest(Configurationconf,Stringpath){try(FileSystemfs=FileSystem.get(conf)){returnfs.exists(newPath(path));}catch(IOExceptione){e.printStackTrace();returnfalse;}}/***追加⽂本内容*/publicstaticvoidappendContentToFile(Configurationconf,Stringcontent,StringremoteFilePath){try(FileSystemfs=FileSystem.get(conf)){PathremotePath=newPath(remoteFilePath);/*创建⼀个⽂件输出流，输出的内容将追加到⽂件末尾*/FSDataOutputStreamout=fs.append(remotePath);out.write(content.getBytes());out.close();}catch(IOExceptione){e.printStackTrace();}}/***追加⽂件内容*/publicstaticvoidappendToFile(Configurationconf,StringlocalFilePath,StringremoteFilePath){PathremotePath=newPath(remoteFilePath);try(FileSystemfs=FileSystem.get(conf);FileInputStreamin=newFileInputStream(localFilePath);){FSDataOutputStreamout=fs.append(remotePath);byte[]data=newbyte[1024];intread=-1;while((read=in.read(data))>0){while((read=in.read(data))>0){out.write(data,0,read);}out.close();}catch(IOExceptione){e.printStackTrace();}}/***移动⽂件到本地移动后，删除源⽂件*/publicstaticvoidmoveToLocalFile(Configurationconf,StringremoteFilePath,StringlocalFilePath){try(FileSystemfs=FileSystem.get(conf)){PathremotePath=newPath(remoteFilePath);PathlocalPath=newPath(localFilePath);fs.moveToLocalFile(remotePath,localPath);}catch(IOExceptione){e.printStackTrace();}}/***创建⽂件*/publicstaticvoidtouchz(Configurationconf,StringremoteFilePath){try(FileSystemfs=FileSystem.get(conf)){PathremotePath=newPath(remoteFilePath);FSDataOutputStreamoutputStream=fs.create(remotePath);outputStream.close();}catch(IOExceptione){e.printStackTrace();}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringremoteFilePath="/user/tiny/text.txt";//HDFS⽂件Stringcontent="新追加的内容\n";Stringchoice="after";//追加到⽂件末尾//Stringchoice="before";//追加到⽂件开头try{/*判断⽂件是否存在*/if(!AppendToFile.test(conf,remoteFilePath)){System.out.println("⽂件不存在:"+remoteFilePath);}else{if(choice.equals("after")){//追加在⽂件末尾AppendToFile.appendContentToFile(conf,content,remoteFilePath);System.out.println("已追加内容到⽂件末尾"+remoteFilePath);}elseif(choice.equals("before")){//追加到⽂件开头/*没有相应的api可以直接操作，因此先把⽂件移动到本地，创建⼀个新的HDFS，再按顺序追加内容*/StringlocalTmpPath="/user/hadoop/tmp.txt";AppendToFile.moveToLocalFile(conf,remoteFilePath,localTmpPath);//移动到本地AppendToFile.touchz(conf,remoteFilePath);//创建⼀个新⽂件AppendToFile.appendContentToFile(conf,content,remoteFilePath);//先写⼊新内容AppendToFile.appendToFile(conf,localTmpPath,remoteFilePath);//再写⼊原来内容remoteFilePath);//再写⼊原来内容System.out.println("已追加内容到⽂件开头:"+remoteFilePath);}}}catch(Exceptione){e.printStackTrace();}}}（9）删除HDFS中指定的⽂件；Shell命令实现：hadoopfs-rm/Test/test1/text1.txt（10）删除HDFS中指定的⽬录，由⽤户指定⽬录中如果存在⽂件时是否删除⽬录；Shell命令实现：hadoopfs-rm-R/Test/test1（11）在HDFS中，将⽂件从源路径移动到⽬的路径。hadoopfs-mvtJAVA实现：importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;importorg.apache.hadoop.fs.FileSystem;importjava.io.*;publicclassMoveFile{/***移动⽂件*/publicstaticbooleanmv(Configurationconf,StringremoteFilePath,StringremoteToFilePath){try(FileSystemfs=FileSystem.get(conf)){PathsrcPath=newPath(remoteFilePath);PathdstPath=newPath(remoteToFilePath);returnfs.rename(srcPath,dstPath);}catch(IOExceptione){e.printStackTrace();returnfalse;}}/***主函数*/publicstaticvoidmain(String[]args){Configurationconf=newConfiguration();conf.set("fs.defaultFS","hdfs://localhost:9000");StringremoteFilePath="hdfs:///user/tiny/text.txt";//源⽂件HDFS路径StringremoteToFilePath="hdfs:///user/tiny/input";//⽬的HDFS路径try{if(MoveFile.mv(conf,remoteFilePath,remoteToFilePath)){System.out.println("将⽂件"+remoteFilePath+"移动到"+remoteToFilePath);}else{System.out.println("操作失败(源⽂件不存在或移动失败)");}}catch(Exceptione){e.printStackTrace();}}}2.编程实现⼀个类“MyFSDataInputStream”，该类继承“org.apache.hadoop.fs.FSDataInputStream”，要求如下：实现按⾏读取HDFS中指定⽂件的⽅法“readLine()”，如果读到⽂件末尾，则返回空，否则返回⽂件⼀⾏的⽂本。JAVA实现：importjava.io.BufferedReader;importjava.io.IOException;importjava.io.InputStream;importjava.io.InputStreamReader;importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.*;publicclassMyFSDataInputStreamextendsFSDataInputStream{/***@paramargs*/public