用到lucene的爬虫的简单实现

小菜鸟我最近研究了一下lucene&＃xff0c;以及前面的爬虫的写法&＃xff0c;我想到能否用lucene写一个站内搜索&＃xff0c;由于我对htmlprase不是很了解&＃xff0c;对字符串的处理有点不行&＃xff0c;但是结果是可以的。

package LuceneSpider;


import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Set;


import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumberTools;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.LockObtainFailedException;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;


public class LuceneSpider {
/**
* 使用种子初始化url队列
*/
private Set visitedUrlSet&＃61;new HashSet();
private LinkedList unvisitedUrlSet&＃61;new LinkedList();
String[] seeds;
String line;
String savepath;
String encoding;
int savenum;
Analyzer analyzer;
public LuceneSpider(String[] seeds,String line,String savepath,int savenum,Analyzer analyzer){
this.seeds&＃61;seeds;
this.line&＃61;line;
this.savepath&＃61;savepath;
this.savenum&＃61;savenum;
this.analyzer&＃61;analyzer;
}
public void init(){
Set seedsSet&＃61;new HashSet();
for(int i&＃61;0;i seedsSet.add(seeds[i]);
}
addToUnvisitedUrlSet(seedsSet);
}
public void run() throws ParserException, HttpException, IOException {
init();
for(int i&＃61;0;i if(IsUnvisitedUrlSetEmpty()&＃61;&＃61;false){
String url&＃61;getFirstFromVisitedUrSet();
catchPages(url);
}
}
}

public void catchPages(String url) throws ParserException, HttpException, IOException{

HttpClient httpClient&＃61;new HttpClient();
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
GetMethod getMethod&＃61;new GetMethod(url);
//生成getmthod对象并设置参数
//设置get请求超时5s
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
//设置请求重试处理
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
//执行http get请求
int statusCode;
statusCode &＃61; httpClient.executeMethod(getMethod);
if(statusCode!&＃61;HttpStatus.SC_OK){
System.err.print("Method faied:"&＃43;url&＃43;getMethod.getStatusLine());
}else{
encoding&＃61;getMethod.getResponseCharSet();
   createIndex(url);
addToVisitedUrlSet(url);
addToUnvisitedUrlSet(getUrls(url));
System.out.println(unvisitedUrlSet.size());
}
}
private void createIndex(String url) throws CorruptIndexException, LockObtainFailedException, IOException, ParserException {
// TODO Auto-generated method stub
String content&＃61;"";
content&＃61;getContentByUrl(url);
Document doc &＃61; new Document();
//文件名称
doc.add(new Field("url", url, Store.YES, Index.NOT_ANALYZED));
//检索到的内容
doc.add(new Field("content",content, Store.YES, Index.ANALYZED));
System.out.println(url);
IndexWriter indexWriter &＃61; new IndexWriter(savepath, analyzer, false,
            MaxFieldLength.LIMITED);
    indexWriter.addDocument(doc);
    indexWriter.close();
}
/*
* 通过url得到网页去除标签后的内容
*/
private String getContentByUrl(String url) throws ParserException {
// TODO Auto-generated method stub
String content&＃61;"";
Parser parser&＃61;new Parser(url);
Node nodes&＃61;null;
int j&＃61;0;
for(NodeIterator iterator&＃61;parser.elements();iterator.hasMoreNodes();){
j&＃43;&＃61;1;
nodes&＃61;iterator.nextNode();
content&＃61;content&＃43;nodes.toPlainTextString().replaceAll(" ","").replaceAll("\n", "");
}
return content;
}
/*
* 解析页面的url
*/
public Set getUrls(String url) throws ParserException {
Set links&＃61;new HashSet();
Parser parser&＃61;null;
parser &＃61; new Parser(url);
parser.setEncoding(encoding);
NodeFilter frameFilter&＃61;new NodeFilter() {
&＃64;Override
public boolean accept(Node node) {
// TODO Auto-generated method stub
if(node.getText().startsWith("frame src&＃61;")){
return true;
}else{
return false;
}
}
};
OrFilter linkFilter&＃61;new OrFilter(new NodeClassFilter(LinkTag.class),frameFilter);
if(parser!&＃61;null){
NodeList list&＃61;parser.extractAllNodesThatMatch(linkFilter);
for(int i&＃61;0;i Node tag&＃61;list.elementAt(i);
if(tag instanceof LinkTag){
LinkTag link&＃61;(LinkTag)tag;
String linkUrl&＃61;link.getLink();
if(frameFilter.accept(tag)){
//处理
String frameTxt&＃61;tag.getText();
int start&＃61;frameTxt.indexOf("src&＃61;");
frameTxt&＃61;frameTxt.substring(start);
int end&＃61;frameTxt.indexOf(" ");
if(end&＃61;&＃61;-1){
end&＃61;frameTxt.indexOf(">");
}
String frameUrl&＃61;frameTxt.substring(5,end-1);
if(LinkFilter(frameUrl))
links.add(frameUrl);
}else{
//处理
if(LinkFilter(linkUrl)){
links.add(linkUrl);
}
}
}
}
}
return links;
}
//爬虫遵循的线索
public boolean LinkFilter(String url){
if(url.startsWith(line)){
return true;
}else{
return false;
}
}

//网页名filter&＃xff0c;不然会出现存储错误
public String getFileNameByUrl(String url,String contentType){
//移除http;
url&＃61;url.substring(7);
//text/html类型
if(contentType.indexOf("html")!&＃61;-1){
url&＃61;url.replaceAll("[\\?/:*|<>\"]", "_")&＃43;".html";
return url;
}else{
return url.replaceAll("[\\?/:*|<>\"]","_")&＃43;"."&＃43;
contentType.substring(contentType.lastIndexOf("/")&＃43;1);
}
}


public void addToVisitedUrlSet(String url){
visitedUrlSet.add(url);
}
public boolean IsUnvisitedUrlSetEmpty(){
boolean isEmpty&＃61;false;
if(unvisitedUrlSet.isEmpty()){
isEmpty&＃61;true;
}
return isEmpty; 
}
public  void addToUnvisitedUrlSet(Set urls){
for (String url : urls) {
if(!isVisited(url)){
unvisitedUrlSet.add(url);
}
}
}
public boolean isVisited(String url){
boolean isVisited&＃61;false;
for (String visitedUrl : visitedUrlSet) {
if(visitedUrl.equals(url)){
isVisited&＃61;true;
}
}
return isVisited;
}
public String getFirstFromVisitedUrSet(){
String url&＃61;unvisitedUrlSet.getFirst().toString();
unvisitedUrlSet.removeFirst();
return url;
}

public void search(String about) throws Exception {
       //请求字段
       //String queryString &＃61; "document";
       //String queryString &＃61; "IndexWriter document a javadoc.txt";


       // 1&＃xff0c;把要搜索的文本解析为 Query
       String[] fields &＃61; { "url", "content" };
       QueryParser queryParser &＃61; new MultiFieldQueryParser(fields, analyzer);
       Query query &＃61; queryParser.parse(about);


       // 2&＃xff0c;进行查询&＃xff0c;从索引库中查找
       IndexSearcher indexSearcher &＃61; new IndexSearcher(savepath);
       Filter filter &＃61; null;
       TopDocs topDocs &＃61; indexSearcher.search(query, filter, 10000);
       System.out.println("总共有【" &＃43; topDocs.totalHits &＃43; "】条匹配结果");


       // 3&＃xff0c;打印结果
       for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
           // 文档内部编号
           int index &＃61; scoreDoc.doc; 
           // 根据编号取出相应的文档
           Document doc &＃61; indexSearcher.doc(index);
           System.out.println("------------------------------");
           System.out.println("url &＃61; " &＃43; doc.get("url"));
//            System.out.println("content &＃61; " &＃43; doc.get("content").replaceAll(" ",""));
       }
   }


}