jieba分词02

import jieba
import sys
reload(sys)
sys.setdefaultencoding(&＃39;utf8&＃39;)def fenci(argv) :filename &＃61; argv[1]f &＃61; open(filename,&＃39;r&＃43;&＃39;)file_list &＃61; f.read()f.close()seg_list &＃61; jieba.cut(file_list,cut_all&＃61;True)tf&＃61;{}for seg in seg_list :#print segseg &＃61; &＃39;&＃39;.join(seg.split())if (seg !&＃61; &＃39;&＃39; and seg !&＃61; "\n" and seg !&＃61; "\n\n") :if seg in tf :tf[seg] &＃43;&＃61; 1else :tf[seg] &＃61; 1f &＃61; open("result.txt","w&＃43;")for item in tf:#print itemf.write(item&＃43;" "&＃43;str(tf[item])&＃43;"\n")f.close()if __name__ &＃61;&＃61; &＃39;__main__&＃39; : fenci(sys.argv)

import os
import jieba
import jieba.posseg as pseg
import sys
import string
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
reload(sys)
sys.setdefaultencoding(&＃39;utf8&＃39;)
#获取文件列表&＃xff08;该目录下放着100份文档&＃xff09;
def getFilelist(argv) :path &＃61; argv[1]filelist &＃61; []files &＃61; os.listdir(path)for f in files :if(f[0] &＃61;&＃61; &＃39;.&＃39;) :passelse :filelist.append(f)return filelist,path
#对文档进行分词处理
def fenci(argv,path) :#保存分词结果的目录sFilePath &＃61; &＃39;./segfile&＃39;if not os.path.exists(sFilePath) : os.mkdir(sFilePath)#读取文档filename &＃61; argvf &＃61; open(path&＃43;filename,&＃39;r&＃43;&＃39;)file_list &＃61; f.read()f.close()#对文档进行分词处理&＃xff0c;采用默认模式seg_list &＃61; jieba.cut(file_list,cut_all&＃61;True)#对空格&＃xff0c;换行符进行处理result &＃61; []for seg in seg_list :seg &＃61; &＃39;&＃39;.join(seg.split())if (seg !&＃61; &＃39;&＃39; and seg !&＃61; "\n" and seg !&＃61; "\n\n") :result.append(seg)#将分词后的结果用空格隔开&＃xff0c;保存至本地。比如"我来到北京清华大学"&＃xff0c;分词结果写入为&＃xff1a;"我 来到 北京 清华大学"f &＃61; open(sFilePath&＃43;"/"&＃43;filename&＃43;"-seg.txt","w&＃43;")f.write(&＃39; &＃39;.join(result))f.close()#读取100份已分词好的文档&＃xff0c;进行TF-IDF计算
def Tfidf(filelist) :path &＃61; &＃39;./segfile&＃xff0f;&＃39;corpus &＃61; [] #存取100份文档的分词结果for ff in filelist :fname &＃61; path &＃43; fff &＃61; open(fname,&＃39;r&＃43;&＃39;)content &＃61; f.read()f.close()corpus.append(content) vectorizer &＃61; CountVectorizer() transformer &＃61; TfidfTransformer()tfidf &＃61; transformer.fit_transform(vectorizer.fit_transform(corpus))word &＃61; vectorizer.get_feature_names() #所有文本的关键字weight &＃61; tfidf.toarray() #对应的tfidf矩阵
sFilePath &＃61; &＃39;./tfidffile&＃39;if not os.path.exists(sFilePath) : os.mkdir(sFilePath)# 这里将每份文档词语的TF-IDF写入tfidffile文件夹中保存for i in range(len(weight)) :print u"--------Writing all the tf-idf in the",i,u" file into ",sFilePath&＃43;&＃39;/&＃39;&＃43;string.zfill(i,5)&＃43;&＃39;.txt&＃39;,"--------"f &＃61; open(sFilePath&＃43;&＃39;/&＃39;&＃43;string.zfill(i,5)&＃43;&＃39;.txt&＃39;,&＃39;w&＃43;&＃39;)for j in range(len(word)) :f.write(word[j]&＃43;" "&＃43;str(weight[i][j])&＃43;"\n")f.close()if __name__ &＃61;&＃61; "__main__" : (allfile,path) &＃61; getFilelist(sys.argv)for ff in allfile :print "Using jieba on "&＃43;fffenci(ff,path)Tfidf(allfile)