中文手机评论情感分析系列（二）

&＃39;&＃39;&＃39;基于词典的评论文本按属性分类&＃39;&＃39;&＃39;import pandas as pd
import re,time
import jieba
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
import numpy as np
import pyltp,osclass TextClass():def __init__(self):self.stopWord=[]&＃39;&＃39;&＃39;分词和词性标注&＃39;&＃39;&＃39;def seg_pos(self,sent):stopWord_Path = &＃39;D:/论文文件/学习文件/情感分析/dict词典/哈工大stopword .txt&＃39;with open(stopWord_Path, &＃39;r&＃39;, encoding=&＃39;utf-8&＃39;) as fr: # 加载停用词for word in fr.readlines():self.stopWord.append(word.strip())line = re.sub(r&＃39;[a-zA-Z0-9]*&＃39;, &＃39;&＃39;, sent)abs_path = &＃39;D:\\LTP\\ltp_data_v3.4.0\\&＃39; # 根目录绝对路径cws_path = os.path.join(abs_path, &＃39;cws.model&＃39;) # 分词库文件路径seg = pyltp.Segmentor() # 分词器对象seg.load(cws_path) # 加载分词语料库cword = seg.segment(line)wordList = list(cword)# print(wordList)word_filter = [word for word in wordList if word not in self.stopWord]pos_model_path = os.path.join(abs_path, &＃39;pos.model&＃39;) # 词性标注模型路径pos = pyltp.Postagger()pos.load(pos_model_path)pos_word = pos.postag(word_filter)pos.release() # 释放实例seg.release()# return list(pos_word),return list(zip(word_filter, list(pos_word)))&＃39;&＃39;&＃39;特征提取&＃39;&＃39;&＃39;def feature_extraction(self,pos):features = []for i in pos:if i[1] in [&＃39;n&＃39;, &＃39;nz&＃39;, &＃39;j&＃39;]:features.append(i[0]) # 直接获得评论中的特征词return features&＃39;&＃39;&＃39;加载词典&＃39;&＃39;&＃39;def openFile(self,path):with open(path,&＃39;r&＃39;,encoding=&＃39;utf-8&＃39;) as f:for word in f.readlines():yield word.strip()&＃39;&＃39;&＃39;文本特征化&＃39;&＃39;&＃39;def text2feature(self,text_list):feature_list=[]for line in text_list:pos=self.seg_pos(line)if len(self.feature_extraction(pos))==0:#如果评论没有特征词的话，那么标记一下为&＃39;none&＃39;，以便于后面对这类无特征文本的处理feature_list.append([&＃39;none&＃39;])else:feature_list.append(self.feature_extraction(pos))return feature_list&＃39;&＃39;&＃39;文本分类&＃39;&＃39;&＃39;def classify(self,feature_list):abs_path = &＃39;D:\\论文文件\\阅读论文\\写论文准备\\字典构建\\手机属性词典\\dictionary_0_3\\&＃39;save_name = [&＃39;相机.txt&＃39;, &＃39;处理器.txt&＃39;, &＃39;价格.txt&＃39;, &＃39;性能.txt&＃39;, &＃39;续航.txt&＃39;,&＃39;外观.txt&＃39;, &＃39;售后.txt&＃39;]c1 = list(self.openFile(abs_path+save_name[0]))c2 = list(self.openFile(abs_path+save_name[1]))c3 = list(self.openFile(abs_path+save_name[2]))c4 = list(self.openFile(abs_path + save_name[3]))c5 = list(self.openFile(abs_path + save_name[4]))c6 = list(self.openFile(abs_path + save_name[5]))c7 = list(self.openFile(abs_path + save_name[6]))dict={&＃39;camera&＃39;:set([]),&＃39;processor&＃39;:set([]),&＃39;price&＃39;:set([]),&＃39;performance&＃39;:set([]),&＃39;endurance&＃39;:set([]),&＃39;appearance&＃39;:set([]),&＃39;serve&＃39;:set([]),&＃39;none&＃39;:set([])}for i in range(len(feature_list)):for j in feature_list[i]:if j in c1:dict[&＃39;camera&＃39;].add(i)if j in c2:dict[&＃39;processor&＃39;].add(i)if j in c3:dict[&＃39;price&＃39;].add(i)if j in c4:dict[&＃39;performance&＃39;].add(i)if j in c5:dict[&＃39;endurance&＃39;].add(i)if j in c6:dict[&＃39;appearance&＃39;].add(i)if j in c7:dict[&＃39;serve&＃39;].add(i)if j==&＃39;none&＃39; :dict[&＃39;none&＃39;].add(i)return dict##########!!!!!!这里缺少一个对于没有特正在7个属性词典的评论的处理，，明天解决！！！（初步想法是建立一个包含所有特征的词典，如果不在则评论放到无属性集合里&＃39;&＃39;&＃39;根据上面分类的结果(索引字典)将原文件里的评论分开，并存储&＃39;&＃39;&＃39;def classify_save(self,index_dict,abs_path,ori_file):# 其中index_dict是分类的结果，为字典索引# abs_path是存储分类文本的绝对地址# ori_file是要分类的评论文本文件，且是DataFrame格式文件#无返回文件，以csv格式存储分类结果keys=index_dict.keys()print(keys)for i in keys:#注意encoding=&＃39;gbk&＃39;,index=False编码格式和去掉列的默认索引DataFrame.ix[]的用法# print(&＃39;关键词&＃39;,i )# print(list(index_dict[i]))# print(ori_file.ix[list(index_dict[i])].comment)ori_file.ix[list(index_dict[i])].to_csv(abs_path+i+&＃39;.csv&＃39;,encoding=&＃39;gbk&＃39;,index=False)&＃39;&＃39;&＃39;集成所有功能的一个函数，直接输入预测数据地址，和保存分类好结果的地址&＃39;&＃39;&＃39;def all(self,pre_data_path,save_abs_path):file=list(pd.read_csv(pre_data_path,sep=&＃39;,&＃39;,encoding=&＃39;GBK&＃39;).comment)text_feature = self.text2feature(file)result = self.classify(text_feature)comment_file = pd.read_csv(pre_data_path, sep=&＃39;,&＃39;, encoding=&＃39;GBK&＃39;)self.classify_save(result, save_abs_path, comment_file)for i in result.keys():#输出各个属性评论占总评论的比例print(i+&＃39;的评论比例:&＃39;, len(result[i]) / len(file))if __name__==&＃39;__main__&＃39; :s=time.time()&＃39;&＃39;&＃39;对Excel中的预测数据按属性分类返回各类别评论索引&＃39;&＃39;&＃39;&＃39;&＃39;&＃39;华为预测数据&＃39;&＃39;&＃39;path_xiaomi = &＃39;D:/machinelearning data/crawlerData/xiaomi6X_pre_JD100.csv&＃39;&＃39;&＃39;&＃39;华为预测数据&＃39;&＃39;&＃39;path_huawei= &＃39;D:/machinelearning data/crawlerData/huaweiP20_pre_JD100.csv&＃39;abs_path_xiaomi=&＃39;D:\\machinelearning data\\crawlerData\\cluster_data\\feature_phone_xiaomi\\&＃39;abs_path_huawei=&＃39;D:\\machinelearning data\\crawlerData\\cluster_data\\feature_phone_huawei\\&＃39;demo=TextClass()#demo.all(path_xiaomi,abs_path_xiaomi)demo.all(path_huawei, abs_path_huawei)e=time.time()print(&＃39;耗时:&＃39;,e-s)