fasttext和gensim训练词向量

"""
https://github.com/facebookresearch/fastText
python版本
https://github.com/salestock/fastText.py
这个是非官方的版本　现在已经不在使用了
官方提供了Python版本　
https://github.com/facebookresearch/fastText/tree/master/python
现在用的都是官方的版本
"""
import jieba
import pandas as pd
import randomcate_dic &＃61; {&＃39;technology&＃39;:1, &＃39;car&＃39;:2, &＃39;entertainment&＃39;:3, &＃39;military&＃39;:4, &＃39;sports&＃39;:5}df_technology &＃61; pd.read_csv("./data/technology_news.csv", encoding&＃61;&＃39;utf-8&＃39;)
df_technology &＃61; df_technology.dropna()df_car &＃61; pd.read_csv("./data/car_news.csv", encoding&＃61;&＃39;utf-8&＃39;)
df_car &＃61; df_car.dropna()df_entertainment &＃61; pd.read_csv("./data/entertainment_news.csv", encoding&＃61;&＃39;utf-8&＃39;)
df_entertainment &＃61; df_entertainment.dropna()df_military &＃61; pd.read_csv("./data/military_news.csv", encoding&＃61;&＃39;utf-8&＃39;)
df_military &＃61; df_military.dropna()df_sports &＃61; pd.read_csv("./data/sports_news.csv", encoding&＃61;&＃39;utf-8&＃39;)
df_sports &＃61; df_sports.dropna()technology &＃61; df_technology.content.values.tolist()[1000:21000]
car &＃61; df_car.content.values.tolist()[1000:21000]
entertainment &＃61; df_entertainment.content.values.tolist()[:20000]
military &＃61; df_military.content.values.tolist()[:20000]
sports &＃61; df_sports.content.values.tolist()[:20000]stopwords&＃61;pd.read_csv("data/stopwords.txt",index_col&＃61;False,quoting&＃61;3,sep&＃61;"\t",names&＃61;[&＃39;stopword&＃39;], encoding&＃61;&＃39;utf-8&＃39;)
stopwords&＃61;stopwords[&＃39;stopword&＃39;].values


"""
fasttext的无监督的词向量训练
https://github.com/facebookresearch/fastText/tree/master/python
"""
import fasttextcate_dic &＃61; {&＃39;technology&＃39;:1, &＃39;car&＃39;:2, &＃39;entertainment&＃39;:3, &＃39;military&＃39;:4, &＃39;sports&＃39;:5}def preprocess_text_unsupervised(content_lines, sentences, category):for line in content_lines:try:segs&＃61;jieba.lcut(line)segs &＃61; filter(lambda x:len(x)>1, segs)segs &＃61; filter(lambda x:x not in stopwords, segs)sentences.append(" ".join(segs))except Exception:print (line)continue
#生成无监督训练数据
sentences &＃61; []preprocess_text_unsupervised(technology, sentences, cate_dic[&＃39;technology&＃39;])
preprocess_text_unsupervised(car, sentences, cate_dic[&＃39;car&＃39;])
preprocess_text_unsupervised(entertainment, sentences, cate_dic[&＃39;entertainment&＃39;])
preprocess_text_unsupervised(military, sentences, cate_dic[&＃39;military&＃39;])
preprocess_text_unsupervised(sports, sentences, cate_dic[&＃39;sports&＃39;])print ("writing data to fasttext unsupervised learning format...")
out &＃61; open(&＃39;unsupervised_train_data.txt&＃39;, &＃39;wb&＃39;)
for sentence in sentences:out.write(sentence.encode(&＃39;utf8&＃39;)&＃43;b"\n")
print("done!") import fasttext# Skipgram model :
skmodel &＃61; fasttext.train_unsupervised(&＃39;unsupervised_train_data.txt&＃39;, model&＃61;&＃39;skipgram&＃39;)# or, cbow model :
cbowmodel &＃61; fasttext.train_unsupervised(&＃39;unsupervised_train_data.txt&＃39;, model&＃61;&＃39;cbow&＃39;)

gensim训练词向量　

import gensim
"""
对比gensim的word2vec
"""model &＃61; gensim.models.Word2Vec(sentences, size&＃61;100, window&＃61;5, min_count&＃61;5, workers&＃61;4)
model.save("gensim_word2vec.model")