开发笔记:lucene统计单词次数(词频)并进行排序

1 public class WordCount {
2 static Directory directory;
3 // 创建分词器
4 static Analyzer analyzer = new IKAnalyzer();
5 static IndexWriterConfig cOnfig= new IndexWriterConfig(analyzer);
6 static IndexWriter writer;
7 static IndexReader reader;
8 static {
9 // 指定索引存放目录以及配置参数
10 try {
11 directory = FSDirectory.open(Paths.get("F:/luceneIndex"));
12 writer = new IndexWriter(directory, config);
13 } catch (IOException e) {
14 e.printStackTrace();
15 }
16 }
17
18 public static void main(String[] args) {
19 indexCreate();
20 Map map = getTotalFreqMap();
21 Map sortMap = sortMapByValue(map);
22 Set> entrySet = sortMap.entrySet();
23 Iterator> iterator = entrySet.iterator();
24 while (iterator.hasNext()) {
25 Entry entry = iterator.next();
26 System.out.println(entry.getKey() + "----" + entry.getValue());
27 }
28
29 }
30
31 /**
32 * 创建索引
33 */
34 public static void indexCreate() {
35 // 文件夹检测(创建索引前要保证目录是空的)
36 File file = new File("f:/luceneIndex");
37 if (!file.exists()) {
38 file.mkdirs();
39 } else {
40 try {
41 file.delete();
42 } catch (Exception e) {
43 e.printStackTrace();
44 }
45 }
46
47 // 将采集的数据封装到Document中
48 Document doc = new Document();
49 FieldType ft = new FieldType();
50 ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
51 ft.setStored(true);
52 ft.setStoreTermVectors(true);
53 ft.setTokenized(true);
54 // ft.setStoreTermVectorOffsets(true);
55 // ft.setStoreTermVectorPositions(true);
56
57 // 读取文件内容(小文件,readFully)
58 File cOntent= new File("f:/qz/twitter.txt");
59 try {
60 byte[] buffer = new byte[(int) content.length()];
61 IOUtils.readFully(new FileInputStream(content), buffer);
62 doc.add(new Field("twitter", new String(buffer), ft));
63 } catch (Exception e) {
64 e.printStackTrace();
65 }
66
67 // 生成索引
68 try {
69 writer.addDocument(doc);
70 // 关闭
71 writer.close();
72
73 } catch (IOException e) {
74 e.printStackTrace();
75 }
76 }
77
78 /**
79 * 获得词频map
80 *
81 * @throws ParseException
82 */
83 public static Map getTotalFreqMap() {
84 Map map = new HashMap();
85 try {
86 reader = DirectoryReader.open(directory);
87 List leaves = reader.leaves();
88 for (LeafReaderContext leafReaderContext : leaves) {
89 LeafReader leafReader = leafReaderContext.reader();
90
91 Terms terms = leafReader.terms("twitter");
92
93 TermsEnum iterator = terms.iterator();
94
95 BytesRef term = null;
96
97 while ((term = iterator.next()) != null) {
98 String text = term.utf8ToString();
99 map.put(text, iterator.totalTermFreq());
100 }
101
102 }
103 reader.close();
104 return map;
105 } catch (IOException e) {
106 e.printStackTrace();
107 }
108 return null;
109 }
110
111 /**
112 * 使用 Map按value进行排序
113 *
114 * @param map
115 * @return
116 */
117 public static Map sortMapByValue(Map oriMap) {
118 if (oriMap == null || oriMap.isEmpty()) {
119 return null;
120 }
121 Map sortedMap = new LinkedHashMap();
122
123 List> entryList = new ArrayList>(oriMap.entrySet());
124 Collections.sort(entryList, new MapValueComparator());
125
126 Iterator> iter = entryList.iterator();
127 Map.Entry tmpEntry = null;
128 while (iter.hasNext()) {
129 tmpEntry = iter.next();
130 sortedMap.put(tmpEntry.getKey(), tmpEntry.getValue());
131 }
132 return sortedMap;
133 }
134 }
135
136 class MapValueComparator implements Comparator> {
137
138 @Override
139 public int compare(Entry me1, Entry me2) {
140 if (me1.getValue() == me2.getValue()) {
141 return 0;
142 }
143 return me1.getValue() > me2.getValue() ? -1 : 1;
144 // return me1.getValue().compareTo(me2.getValue());
145 }
146 }