JAVA网络爬虫实现3

《JAVA网络爬虫实现3》由会员上传分享，免费在线阅读，更多相关内容在教育资源-天天文库。

1、package爬虫;importjava.io.*;importorg.apache.commons.httpclient.DefaultHttpMethodRetryHandler;importorg.apache.commons.httpclient.HttpClient;importorg.apache.commons.httpclient.HttpStatus;importorg.apache.commons.httpclient.methods.GetMethod;importorg.apache.commons.httpclient.params.Ht

2、tpMethodParams;publicclassDownLoadFile{publicStringgetFileNameByUrl(Stringurl,StringcontentType){url=url.substring(7);if(contentType.indexOf("html")!=-1){url=url.replace("[\?/:*

3、<>"]","_")+".html";Stringtemp=url.replaceAll("[\?/:*

4、<>"]","_");//System.out.println("modified:"+url.re

5、placeAll("[\?/:*

6、<>"]","_"));returntemp;}else{Stringtemp=url.replaceAll("[\?/:*

7、<>"]","_")+"."+contentType.substring(contentType.lastIndexOf("/")+1);url=url.replaceAll("[\?/:*

8、<>"]","_")+"."+contentType.substring(contentType.lastIndexOf("/")+1);//System.out.printf("modified:"+ur

9、l);returntemp;}}publicvoidsaveToLocal(byte[]data,StringfilePath){try{DataOutputStreamout=newDataOutputStream(newFileOutputStream(newFile(filePath)));for(inti=0;i

10、ile(Stringurl){StringfilePath=null;HttpClienthttpClient=newHttpClient();httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);GetMethodgetMethod=newGetMethod(url);getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,5000);getMethod.getParams().setParam

11、eter(HttpMethodParams.RETRY_HANDLER,newDefaultHttpMethodRetryHandler());try{intstatusCode=httpClient.executeMethod(getMethod);if(statusCode!=HttpStatus.SC_OK){System.err.println("Methodfailed:"+getMethod.getStatusLine());filePath=null;}byte[]responseBody=getMethod.getResponseBody();St

12、ringfileName=null;if(url.length()>100)fileName=url.substring(0,100);elsefileName=url;StringfileType=getMethod.getResponseHeader("Content-Type").getValue();//System.out.println("文件名和文件类型分别是："+fileName+"t"+fileType);filePath="D:\testFile\"+getFileNameByUrl(fileName,getMethod.getRespo

13、nseHe