Java使用HtmlParser实现简单的网络爬虫

Java使用HtmlParser实现简单的网络爬虫目的Java使用HtmlParser抓取网页数据并解析。效果图例如：抓取12306网站中货物运价率的数据貝Console胡占粘"rv巳『寸菇D巳Hug<terminated>JavaCrawler[JavaApplication]D:\ProgramFiIes\Java\jdkl.7.021\bin\javaw.exe(2013-11-27上午10:04:31)基价工标连£ 基价2单位基价2标准7.40元■/吨公里0.05657.90元■/吨公里0.065110.50元■/吨公里0.070013.86元/吨公里0・075315.40]元■/吨公里0.084922.20元■/吨公里0.1146^nbsp;元/轴公里0・402516.70元/吨公里0.11340.168元/苗千克公里0.000860.235元皿千克公里0.00120387.50元/箱公里1.7325527.60元/箱公里2.3562吨江屯屯屯屯屯屯斗基元元元元元元亦千千00」.目一目11糜糜

卸7L/卸7L/4567机械冷藏车212220英尺箱40英尺箱名箱箱别车车车车车车车车担担爲廳类整整整整整整整整零霧集集packagecom.zc.test;import.HttpURLConnection;import.URL;importorg.htmlparser.Node;importorg.htmlparser.NodeFilter;importorg.htmlparser.Parser;importorg.htmlparserfilters.AndFilter;importorg.htmlparserfilters.HasAttributeFilter;importorg.htmlparserfilters.TagNameFilter;importorg.htmlparser.nodes.TagNode;importorg.htmlparser.tags.TableColumn;importorg.htmlparser.tags.TableRow;importorg.htmlparser.tags.TableTag;importorg.htmlparser.util.NodeList;©Descriptionjava爬虫测试类©author 张川(cr10210206@163.com)@data 2013-11-2710:01:13

*/publicclassJavaCrawler{/***测试**/publicstaticvoidmain(String[]args){JavaCrawlertest=newJavaCrawler();//货物运价率test.getGoodFareData("/hyinfo/action/JgxxAction_hwyjl?lx=00");//危险品运输栏目 危险品栏目内容//test.getDangerDivData("/hyinfo/page/home-freightProduction-Dangerous_index");//超限超重栏目超限超重栏目内容//test.getDangerDivData("/hyinfo/page/home-freightProduction-overweight_index");}/***获取Parser对象@paramurl@paramurl*/网络地址publicstaticParsergetParser(Stringurl){try{newParserparser=newParser((HttpURLConnection)newURL(url).openConnection());//数据源parser.setEncoding("UTF-8");//编码returnparser;}catch(Exceptione){e.printStackTrace();}returnnull;/***获取货物运价率数据**@paramurl 网络地址*/publicvoidgetGoodFareData(Stringurl){try{//解析器Parserparser=getParser(url);//标签过滤，第一个参数代表过滤的哪个标签，第二个参数代表过滤标签的哪个属性NodeFilterdivFilter=new AndFilter(newTagNameFilter(”table"),newHasAttributeFilter("id","T1"));//把过滤后的内容加入NodeList里进行遍历NodeList nodeList =parser.extractAllNodesThatMatch(divFilter);TableTagtableTag=(TableTag)nodeList.elementAt(0)//获取Table对象TableRow[]tableRows=tableTag.getRows();//获取行数组Stringandstr=" ";Systemout.println(”类别名称 运价号 基价1单位 基价1标准 基价2单位 基价2标准");for(inti=2;i<tableRows.length;i++){TableRowtableRow=tableRows[i];TableColumn[]tableColumns=tableRow.getColumns();//获取列数组TOC\o"1-5"\h\zString typeName =tableColumns[0].getStringText().replace(andstr,"”).trim();String fareid =tableColumns[1].getStringText().replace(andstr,"”).trim();String price1Unit =tableColumns[2].getStringText().replace(andstr,"”).trim();String price1St =tableColumns[3].getStringText().replace(andstr,"”).trim();String price2Unit =tableColumns[4].getStringText().replace(andstr,"”).trim();String price2St =tableColumns[5].getStringText().replace(andstr,"”).trim();System.out.println(typeName+" "+fareid+""+price1Unit+" "+price1St+" "+price2Unit+"

+price2St);}}catch(Exceptione){e.printStackTrace();*获取危险品运输栏目数据*@paramurl网络地址*/publicvoidgetDangerDivData(Stringurl){try{//解析器Parserparser=getParser(url);//标签过滤，第一个参数代表过滤的哪个标签，第二个参数代表过滤标签的哪个属性NodeFilterdivFilter=new AndFilter(newTagNameFilter("div"),newHasAttributeFilter("class","cleft"));//把过滤后的内容加入NodeList里进行遍历nodeListNodeListnodeListparser.extractAllNodesThatMatch(divFilter);Nodenode=nodeList.elementAt(0);NodeListchildNodeList=node.getChildren();if(childNodeList!=null&&childNodeList.size()>0){System.out.println("\n\n栏目标识 栏目名称”)；for(inti=0;i<childNodeList.size();i++){NodechildNode=childNodeList.elementAt(i);if(childNodeinstanceofTagNode){TagNodetag=(TagNode)childNode;Stringid=tag.getAttribute("id");Stringvalue=tag.toPlainTextString();if(id!=null){System.out.println(id+" "+value);id;\n");url=url.substring(0,url.index0f("_"id;\n");System.out.println("["+value+"]的内容如下：this.getDangerContent(url);System.out.println( ");}}}}}catch(Exceptione){e.printStackTrace();}}*危险品栏目内容*@paramurl网络地址*/publicvoidgetDangerContent(Stringurl){try{//解析器Parserparser=getParser(url);//标签过滤，第一个参数代表过滤的哪个标签，第二个参数代表过滤标签的哪个属性NodeFilterdivFilter=new AndFilter(newTagNameFilter("div"),newHasAttribu