fasttext文本分类python实现_基于FastText进行文本分类

liunx版本下操作&＃xff1a;

$ git clone https://github.com/facebookresearch/fastText.git

$ cd fastText

$ pip install .

安装成功后的导入&＃xff1a;

新建test.py文件&＃xff0c;写入&＃xff1a;

import fastText.FastText as fasttext(可能会瞟红线)

新增&＃xff1a;最近发现fasttext的github更新了&＃xff0c;引入方式发生了变化&＃xff0c;如果上述引入报错&＃xff0c;改成 import fasttext.FastText as fasttext

新增&＃xff1a;现在安装直接 pip install fasttext&＃xff0c;导入直接 import fasttext 就行

保存后退出并运行&＃xff1a;

python3 test.py

没报错说明安装成功第二步&＃xff1a;准备数据集我这里用的是清华的新闻数据集(由于完整数据集较大&＃xff0c;这里只取部分数据)

数据链接&＃xff1a;点击获取网盘数据 提取码&＃xff1a;byoi(data.txt为数据集&＃xff0c;stopwords.txt为停用词)

下载好后的数据格式为&＃xff1a;

对应的标签分别为(由于只是用小部分数据&＃xff0c;所以data.txt只包含部分标签)&＃xff1a;

mapper_tag &＃61; {

&＃39;财经&＃39;: &＃39;Finance&＃39;,

&＃39;彩票&＃39;: &＃39;Lottery&＃39;,

&＃39;房产&＃39;: &＃39;Property&＃39;,

&＃39;股票&＃39;: &＃39;Shares&＃39;,

&＃39;家居&＃39;: &＃39;Furnishing&＃39;,

&＃39;教育&＃39;: &＃39;Education&＃39;,

&＃39;科技&＃39;: &＃39;Technology&＃39;,

&＃39;社会&＃39;: &＃39;Sociology&＃39;,

&＃39;时尚&＃39;: &＃39;Fashion&＃39;,

&＃39;时政&＃39;: &＃39;Affairs&＃39;,

&＃39;体育&＃39;: &＃39;Sports&＃39;,

&＃39;星座&＃39;: &＃39;Constellation&＃39;,

&＃39;游戏&＃39;: &＃39;Game&＃39;,

&＃39;娱乐&＃39;: &＃39;Entertainment&＃39;

}第三步&＃xff1a;数据预处理由于data.txt已经经过了分词和去停用词的处理&＃xff0c;所以这里只需要对数据进行切割为训练集和测试集即可。

分词和去停用词的工具代码(运行时不需要执行此部分代码)&＃xff1a;

import re

from types import MethodType, FunctionType

import jieba

def clean_txt(raw):

fil &＃61; re.compile(r"[^0-9a-zA-Z\u4e00-\u9fa5]&＃43;")

return fil.sub(&＃39; &＃39;, raw)

def seg(sentence, sw, apply&＃61;None):

if isinstance(apply, FunctionType) or isinstance(apply, MethodType):

sentence &＃61; apply(sentence)

return &＃39; &＃39;.join([i for i in jieba.cut(sentence) if i.strip() and i not in sw])

def stop_words():

with open(&＃39;stop_words.txt&＃39;, &＃39;r&＃39;, encoding&＃61;&＃39;utf-8&＃39;) as swf:

return [line.strip() for line in swf]

# 对某个sentence进行处理&＃xff1a;

content &＃61; &＃39;上海天然橡胶期价周三再创年内新高&＃xff0c;主力合约突破21000元/吨重要关口。&＃39;

res &＃61; seg(content.lower().replace(&＃39;\n&＃39;, &＃39;&＃39;), stop_words(), apply&＃61;clean_txt)切割数据(这里我是先将txt文件转换成csv文件&＃xff0c;方便后面的计算)

from random import shuffle

import pandas as pd

class _MD(object):

mapper &＃61; {

str: &＃39;&＃39;,

int: 0,

list: list,

dict: dict,

set: set,

bool: False,

float: .0

}

def __init__(self, obj, default&＃61;None):

self.dict &＃61; {}

assert obj in self.mapper, \

&＃39;got a error type&＃39;

self.t &＃61; obj

if default is None:

return

assert isinstance(default, obj), \

f&＃39;default ({default}) must be {obj}&＃39;

self.v &＃61; default

def __setitem__(self, key, value):

self.dict[key] &＃61; value

def __getitem__(self, item):

if item not in self.dict and hasattr(self, &＃39;v&＃39;):

self.dict[item] &＃61; self.v

return self.v

elif item not in self.dict:

if callable(self.mapper[self.t]):

self.dict[item] &＃61; self.mapper[self.t]()

else:

self.dict[item] &＃61; self.mapper[self.t]

return self.dict[item]

return self.dict[item]

def defaultdict(obj, default&＃61;None):

return _MD(obj, default)

class TransformData(object):

def to_csv(self, handler, output, index&＃61;False):

dd &＃61; defaultdict(list)

for line in handler:

label, content &＃61; line.split(&＃39;,&＃39;, 1)

dd[label.strip(&＃39;__label__&＃39;).strip()].append(content.strip())

df &＃61; pd.DataFrame()

for key in dd.dict:

col &＃61; pd.Series(dd[key], name&＃61;key)

df &＃61; pd.concat([df, col], axis&＃61;1)

return df.to_csv(output, index&＃61;index, encoding&＃61;&＃39;utf-8&＃39;)

def split_train_test(source, auth_data&＃61;False):

if not auth_data:

train_proportion &＃61; 0.8

else:

train_proportion &＃61; 0.98

basename &＃61; source.rsplit(&＃39;.&＃39;, 1)[0]

train_file &＃61; basename &＃43; &＃39;_train.txt&＃39;

test_file &＃61; basename &＃43; &＃39;_test.txt&＃39;

handel &＃61; pd.read_csv(source, index_col&＃61;False, low_memory&＃61;False)

train_data_set &＃61; []

test_data_set &＃61; []

for head in list(handel.head()):

train_num &＃61; int(handel[head].dropna().__len__() * train_proportion)

sub_list &＃61; [f&＃39;__label__{head} , {item.strip()}\n&＃39; for item in handel[head].dropna().tolist()]

train_data_set.extend(sub_list[:train_num])

test_data_set.extend(sub_list[train_num:])

shuffle(train_data_set)

shuffle(test_data_set)

with open(train_file, &＃39;w&＃39;, encoding&＃61;&＃39;utf-8&＃39;) as trainf,\

open(test_file, &＃39;w&＃39;, encoding&＃61;&＃39;utf-8&＃39;) as testf:

for tds in train_data_set:

trainf.write(tds)

for i in test_data_set:

testf.write(i)

return train_file, test_file

# 转化成csv

td &＃61; TransformData()

handler &＃61; open(&＃39;data.txt&＃39;)

td.to_csv(handler, &＃39;data.csv&＃39;)

handler.close()

# 将csv文件切割&＃xff0c;会生成两个文件(data_train.txt和data_test.txt)

train_file, test_file &＃61; split_train_test(&＃39;data.csv&＃39;, auth_data&＃61;True)第四步&＃xff1a;训练模型

import fastText.FastText as fasttext

def train_model(ipt&＃61;None, opt&＃61;None, model&＃61;&＃39;&＃39;, dim&＃61;100, epoch&＃61;5, lr&＃61;0.1, loss&＃61;&＃39;softmax&＃39;):

np.set_printoptions(suppress&＃61;True)

if os.path.isfile(model):

classifier &＃61; fasttext.load_model(model)

else:

classifier &＃61; fasttext.train_supervised(ipt, label&＃61;&＃39;__label__&＃39;, dim&＃61;dim, epoch&＃61;epoch,

lr&＃61;lr, wordNgrams&＃61;2, loss&＃61;loss)

"""训练一个监督模型, 返回一个模型对象&＃64;param input: 训练数据文件路径&＃64;param lr: 学习率&＃64;param dim: 向量维度&＃64;param ws: cbow模型时使用&＃64;param epoch: 次数&＃64;param minCount: 词频阈值, 小于该值在初始化时会过滤掉&＃64;param minCountLabel: 类别阈值&＃xff0c;类别小于该值初始化时会过滤掉&＃64;param minn: 构造subword时最小char个数&＃64;param maxn: 构造subword时最大char个数&＃64;param neg: 负采样&＃64;param wordNgrams: n-gram个数&＃64;param loss: 损失函数类型, softmax, ns: 负采样, hs: 分层softmax&＃64;param bucket: 词扩充大小, [A, B]: A语料中包含的词向量, B不在语料中的词向量&＃64;param thread: 线程个数, 每个线程处理输入数据的一段, 0号线程负责loss输出&＃64;param lrUpdateRate: 学习率更新&＃64;param t: 负采样阈值&＃64;param label: 类别前缀&＃64;param verbose: ??&＃64;param pretrainedVectors: 预训练的词向量文件路径, 如果word出现在文件夹中初始化不再随机&＃64;return model object"""

classifier.save_model(opt)

return classifier

dim &＃61; 100

lr &＃61; 5

epoch &＃61; 5

model &＃61; f&＃39;data_dim{str(dim)}_lr0{str(lr)}_iter{str(epoch)}.model&＃39;

classifier &＃61; train_model(ipt&＃61;&＃39;data_train.txt&＃39;,

opt&＃61;model,

model&＃61;model,

dim&＃61;dim, epoch&＃61;epoch, lr&＃61;0.5

)

result &＃61; classifier.test(&＃39;data_test.txt&＃39;)

print(result)

# 整体的结果为(测试数据量&＃xff0c;precision&＃xff0c;recall)&＃xff1a;

(9885, 0.9740010116337886, 0.9740010116337886)可以看出结果相当高&＃xff0c;由于上面是将整体作为测试&＃xff0c;fasttext只给出整体的结果&＃xff0c;precision和recall是相同的&＃xff0c;下面我们测试每个标签的precision、recall和F1值。

def cal_precision_and_recall(file&＃61;&＃39;data_test.txt&＃39;):

precision &＃61; defaultdict(int, 1)

recall &＃61; defaultdict(int, 1)

total &＃61; defaultdict(int, 1)

with open(file) as f:

for line in f:

label, content &＃61; line.split(&＃39;,&＃39;, 1)

total[label.strip().strip(&＃39;__label__&＃39;)] &＃43;&＃61; 1

labels2 &＃61; classifier.predict([seg(sentence&＃61;content.strip(), sw&＃61;&＃39;&＃39;, apply&＃61;clean_txt)])

pre_label, sim &＃61; labels2[0][0][0], labels2[1][0][0]

recall[pre_label.strip().strip(&＃39;__label__&＃39;)] &＃43;&＃61; 1

if label.strip() &＃61;&＃61; pre_label.strip():

precision[label.strip().strip(&＃39;__label__&＃39;)] &＃43;&＃61; 1

print(&＃39;precision&＃39;, precision.dict)

print(&＃39;recall&＃39;, recall.dict)

print(&＃39;total&＃39;, total.dict)

for sub in precision.dict:

pre &＃61; precision[sub] / total[sub]

rec &＃61; precision[sub] / recall[sub]

F1 &＃61; (2 * pre * rec) / (pre &＃43; rec)

print(f"{sub.strip(&＃39;__label__&＃39;)} precision: {str(pre)} recall: {str(rec)} F1: {str(F1)}")结果&＃xff1a;

precision {&＃39;Technology&＃39;: 983, &＃39;Education&＃39;: 972, &＃39;Shares&＃39;: 988, &＃39;Affairs&＃39;: 975, &＃39;Entertainment&＃39;: 991, &＃39;Financ&＃39;: 982, &＃39;Furnishing&＃39;: 975, &＃39;Gam&＃39;: 841, &＃39;Sociology&＃39;: 946, &＃39;Sports&＃39;: 978}

recall {&＃39;Technology&＃39;: 992, &＃39;Education&＃39;: 1013, &＃39;Shares&＃39;: 1007, &＃39;Affairs&＃39;: 995, &＃39;Entertainment&＃39;: 1022, &＃39;Financ&＃39;: 1001, &＃39;Furnishing&＃39;: 997, &＃39;Gam&＃39;: 854, &＃39;Sociology&＃39;: 1025, &＃39;Sports&＃39;: 989}

total {&＃39;Technology&＃39;: 1001, &＃39;Education&＃39;: 1001, &＃39;Shares&＃39;: 1001, &＃39;Affairs&＃39;: 1001, &＃39;Entertainment&＃39;: 1001, &＃39;Financ&＃39;: 1001, &＃39;Furnishing&＃39;: 1001, &＃39;Gam&＃39;: 876, &＃39;Sociology&＃39;: 1001, &＃39;Sports&＃39;: 1001, &＃39;Property&＃39;: 11}

Technology precision: 0.9820179820179821 recall: 0.9909274193548387 F1: 0.9864525840441545

Education precision: 0.971028971028971 recall: 0.9595261599210266 F1: 0.9652432969215492

Shares precision: 0.987012987012987 recall: 0.9811320754716981 F1: 0.9840637450199202

Affairs precision: 0.974025974025974 recall: 0.9798994974874372 F1: 0.9769539078156312

Entertainment precision: 0.99000999000999 recall: 0.9696673189823874 F1: 0.9797330696984675

Financ precision: 0.981018981018981 recall: 0.981018981018981 F1: 0.981018981018981

Furnishing precision: 0.974025974025974 recall: 0.9779338014042126 F1: 0.975975975975976

Gam precision: 0.9600456621004566 recall: 0.9847775175644028 F1: 0.9722543352601155

Sociology precision: 0.945054945054945 recall: 0.9229268292682927 F1: 0.9338598223099703

Sports precision: 0.977022977022977 recall: 0.9888776541961577 F1: 0.9829145728643216

可以看出结果非常可观&＃xff0c;fasttext很强大...整合后的代码&＃xff1a;

def main(source):

basename &＃61; source.rsplit(&＃39;.&＃39;, 1)[0]

csv_file &＃61; basename &＃43; &＃39;.csv&＃39;

td &＃61; TransformData()

handler &＃61; open(source)

td.to_csv(handler, csv_file)

handler.close()

train_file, test_file &＃61; split_train_test(csv_file)

dim &＃61; 100

lr &＃61; 5

epoch &＃61; 5

model &＃61; f&＃39;data/data_dim{str(dim)}_lr0{str(lr)}_iter{str(epoch)}.model&＃39;

classifier &＃61; train_model(ipt&＃61;train_file,

opt&＃61;model,

model&＃61;model,

dim&＃61;dim, epoch&＃61;epoch, lr&＃61;0.5

)

result &＃61; classifier.test(test_file)

print(result)

cal_precision_and_recall(test_file)

if __name__ &＃61;&＃61; &＃39;__main__&＃39;:

main(&＃39;data.txt&＃39;)