JAVA基于Lucene实现建立索引以及检索

Step 1 利用Lucene建立索引

数据库中存放了数据集，对应上万个TXT的文件，表FILE含有两个字段，分别为path和content，依次保存文件的路径和内容。这里，利用Lucene对这些数据建立索引。

 

package pretreatment; import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.util.Date; import java.util.List; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; /** * BuildIndex用于创建索引，数据由数据库读取，创建的索引目录为INDEX_PATH * @author Tony * */ public class BuildIndex{ //索引存储路径 private String INDEX_PATH = "D://index"; public void createIndex(){ try{ //起始时间 Date start = new Date(); //创建分析器 StandardAnalyzer standardAnalyzer = new StandardAnalyzer(); //创建IndexWriter IndexWriter indexWriter = new IndexWriter(INDEX_PATH, standardAnalyzer, true); //参数设置 indexWriter.setMergeFactor(100); indexWriter.setMaxBufferedDocs(100); indexWriter.setMaxFieldLength(1000); //从数据库获得数据对象 Connection conn = BuildIndex.creatConn(); int st = 0; int en = 49; int num = 1; while(en<=15900){ List<FileInfo> fileList = FileInfoController.getFileInfo(st,en,conn); st = en+1; en = st+49; for (int i = 0; i < fileList.size(); i++) { FileInfo fInfo = (FileInfo) fileList.get(i); //System.out.println(fInfo.getContent()); Document doc = new Document(); Field field = new Field("fileName",fInfo.getFileName(),Field.Store.YES,Field.Index.ANALYZED); doc.add(field); field = new Field("content",fInfo.getContent(),Field.Store.COMPRESS,Field.Index.ANALYZED,Field.TermVector.WITH_POSITIONS_OFFSETS); doc.add(field); indexWriter.addDocument(doc); System.out.println("****已完成对**"+fInfo.getFileName()+"**文件的分析****"+"ID:"+num); num++; } //fileList = null; //System.gc(); } //执行优化 indexWriter.optimize(); //关闭indexWriter indexWriter.close(); //关闭数据库连接 conn.close(); System.out.println("****Connection Successful Shut Down****"); //结束时间 Date end = new Date(); System.out.println("****索引创建完毕，共耗时: " + (end.getTime() - start.getTime()) + "毫秒"); }catch(Exception e){ System.out.println(e.toString()); System.out.println("Unexpected Error Occoured ! Please restart the program later "); } } //建立数据库连接 private static Connection creatConn(){ Connection conn = null; try { Class.forName("com.mysql.jdbc.Driver").newInstance(); } catch (InstantiationException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IllegalAccessException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/txtsource?useUnicode=true&characterEncoding=utf-8", "root", "root"); System.out.println("*****Set up Connection Successed!******"); } catch (SQLException e) { // TODO Auto-generated catch block e.printStackTrace(); } return conn; } public static void main(String[] args){ BuildIndex b = new BuildIndex(); b.createIndex(); } } 

Step 2 查询索引

对于已经建立好的索引，下一步便是查询，这里，将查询语句写在ACTION中，然后将结果返回JSP进行显示。

package action; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.StringReader; import java.util.ArrayList; import java.util.Date; import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.Hits; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.QueryScorer; import org.apache.lucene.search.highlight.SimpleFragmenter; import org.apache.lucene.search.highlight.SimpleHTMLFormatter; import org.apache.struts2.ServletActionContext; import javaBean.ResultInfoBean; import com.opensymphony.xwork2.ActionContext; import com.opensymphony.xwork2.ActionSupport; public class SearchAction extends ActionSupport{ /** * 接受查询输入 */ private static final long serialVersionUID = 1L; private String keyWord; private int maxResults; private String path; private String content; private long costTime; private int resultSize; private String INDEX_PATH = "D://index"; private StandardAnalyzer analyzer = new StandardAnalyzer(); List<ResultInfoBean> lfi = new ArrayList<ResultInfoBean>(); public List<ResultInfoBean> getLfi() { return lfi; } public void setLfi(List<ResultInfoBean> lfi) { this.lfi = lfi; } public long getCostTime() { return costTime; } public void setCostTime(long costTime) { this.costTime = costTime; } public int getResultSize() { return resultSize; } public void setResultSize(int resultSize) { this.resultSize = resultSize; } public String getPath() { return path; } public void setPath(String path) { this.path = path; } public String getContent() { return content; } public void setContent(String content) { this.content = content; } public String getKeyWord() { return keyWord; } public void setKeyWord(String keyWord) { this.keyWord = keyWord; } public int getMaxResults() { return maxResults; } public void setMaxResults(int maxResults) { this.maxResults = maxResults; } public String execute() throws Exception { // TODO Auto-generated method stub try { //开始时间 Date start = new Date(); //Begin IndexSearcher searcher = new IndexSearcher(INDEX_PATH); QueryParser queryParse = new QueryParser("content", analyzer); Query query = queryParse.parse(keyWord); Hits hits = searcher.search(query); //高亮显示设置 //SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<span class=/"highlight/">","</span>"); SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<span class=/"highlight/">","</span>"); Highlighter highlighter = new Highlighter(simpleHTMLFormatter,new QueryScorer(query)); highlighter.setTextFragmenter(new SimpleFragmenter(50));//这个100是指定关键字字符串的context的长度，你可以自己设定，因为不可能返回整篇正文内容 boolean bool = true; if(hits.length()==0){ bool = false; String ms = "没有找到相关的记录！"; }else{ for(int i=0;i<hits.length();i++){ Document doc = hits.doc(i); //System.out.println("查询结果:"+doc.get("fileName")); //System.out.println(doc.get("content")); //高亮出显示 TokenStream tokenStream =analyzer.tokenStream("content", new StringReader(doc.get("content"))); content = highlighter.getBestFragment(tokenStream,doc.get("content")); //System.out.println(content); String path = doc.get("fileName"); ResultInfoBean ri = new ResultInfoBean(); ri.setPath(path); ri.setContent(content); lfi.add(ri); } } //结束时间 Date end = new Date(); long timeOfSearch = end.getTime() - start.getTime(); System.out.println("搜索耗时:"+timeOfSearch); costTime=timeOfSearch;//共耗时 resultSize = hits.length();//共找到条记录 // HttpServletRequest request = ServletActionContext.getRequest(); request.setAttribute("sl", 0); request.setAttribute("numPerPage", maxResults); HttpSession session = request.getSession(); session.setAttribute("lfi", lfi); } catch (CorruptIndexException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return SUCCESS; } }  

Step 3 显示与分页

对于分页机制，我选择的是将用户查询的结果保存在SESSION中，每次在SESSION中取相应大小的结果，这样做的结果是用户在点击查询的时候，速度相对比较慢，但分页时就不需要重复查询。当然了，个人认为这是个比较stupid的方法，但一时也没想到更好的方法，只好将就了……

<%@ page language="java" import ="java.util.*" import="javaBean.*;" pageEncoding="UTF-8"%> <%@ taglib prefix="s" uri="/struts-tags" %> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html> <head> <title>Search Result</title> <link rel="stylesheet" type="text/css" href="css/styles.css" mce_href="css/styles.css"> <mce:style type="text/css"><!-- .highlight{ color: #C71585; font-weight: bold;} --></mce:style><style type="text/css" mce_bogus="1"> .highlight{ color: #C71585; font-weight: bold;} </style> </head> <s:include value="head.jsp"/> <body> <HR style="FILTER: alpha(opacity=100,finishopacity=0,style=1)" mce_style="FILTER: alpha(opacity=100,finishopacity=0,style=1)" width="100%" color=#0000CD SIZE=1> <% //初始化全局变量 int totalNum = 0; //总共记录条数 int numPerPage = 0; //每页显示记录条数 int maxPageNum = 0; //本次搜索最大页数 int curPage = 1; //当前页 int startLocation = 0; //起始位置 List<ResultInfoBean> searchresult = new ArrayList<ResultInfoBean>(); searchresult = (List<ResultInfoBean>) session.getAttribute("lfi"); startLocation = (Integer)request.getAttribute("sl"); numPerPage = (Integer)request.getAttribute("numPerPage"); totalNum = searchresult.size(); maxPageNum=((totalNum%numPerPage) == 0)?totalNum/numPerPage:(totalNum/numPerPage+1); curPage=startLocation/numPerPage + 1; %> <% if(curPage<maxPageNum){ System.out.println("startLocation="+startLocation); for(int i=startLocation; i<startLocation+numPerPage; i++){ ResultInfoBean result = (ResultInfoBean) searchresult.get(i); out.print("<p><b><font color='#0000CD'>"+result.getPath()+"</font></b>"); out.println("<p>"+result.getContent()); } }else{ System.out.println("totalNum="+totalNum); for(int i=startLocation; i<totalNum; i++){ ResultInfoBean result = (ResultInfoBean) searchresult.get(i); out.print("<p><b><font color='#0000CD'>"+result.getPath()+"</font></b>"); out.println("<p>"+result.getContent()); } } %> <br> <br> <p align="center"> <% if (curPage > 1 ) { out.println("<a href="PagingControl.action?startLocation="+(startLocation-numPerPage)); out.println("" mce_href="PagingControl.action?startLocation="+(startLocation-numPerPage)); out.println("">上一页</a>"); } if(maxPageNum > 10) { int totalNumShow = curPage + 9; int startLocationPage = 0; if( totalNumShow > maxPageNum) { totalNumShow = maxPageNum; } if( maxPageNum > 11 && curPage > 11) { startLocationPage = curPage - 11; } for(int i=startLocationPage;i<totalNumShow;i++) { if(i+1 == curPage) { %> [<%=i+1%>]  <% } else { %><a href="PagingControl.action?startLocation=<%=i*numPerPage%>"> [<%=i+1 %>] </a><% } } } else { for( int i=0;i<maxPageNum;i++) { if( i+1 == curPage) { %> [<%=i+1 %>]  <% } else { %><a href="PagingControl.action?startLocation=<%=i*numPerPage%>"> [<%=i+1 %>] </a><% } } } if ( curPage < maxPageNum) { out.println("<a href="PagingControl.action?startLocation="+(startLocation+numPerPage)); out.println("" mce_href="PagingControl.action?startLocation="+(startLocation+numPerPage)); out.println("">下一页</a>"); } %> </p> </body> </html>  

 

这个程序的效果就跟Google差不多，只不过查询只限于本地数据库中已存好的数据。