word2vecpython实现_NLPword2vec的python实现

import logging

from gensim.models import word2vec

import multiprocessing

# 配置日志

logging.basicConfig(format&＃61;&＃39;%(asctime)s - %(levelname)s - %(message)s&＃39;, level&＃61;logging.INFO)

# 评论的二维数组&＃xff0c;其中的一维数组是每条评论分词之后的list

sentences &＃61; [[&＃39;电池&＃39;, &＃39;续航&＃39;, &＃39;不错&＃39;], [&＃39;手机&＃39;, &＃39;电池&＃39;, &＃39;容量&＃39;, &＃39;大&＃39;]]

# 模型路径

model_path &＃61; &＃39;../model/study_w2v&＃39;

# 词向量训练&＃xff0c; 并保存模型

model &＃61; word2vec.Word2Vec(size&＃61;128, min_count&＃61;3, window&＃61;5, workers&＃61;multiprocessing.cpu_count(), sg&＃61;1)

model.build_vocab(sentences)

model.train(sentences, total_examples&＃61;model.corpus_count, epochs&＃61;model.epochs)

model.save(model_path)

# 加载模型

model &＃61; word2vec.Word2Vec.load(model_path)

# 查看模型的词表中词 频度和索引

for key, value in model.wv.vocab.items(): # model.wv.vocab是一个dict

print(key, value)

# 判断模型词表中是否存在某个词

print(&＃39;手机&＃39; in model.wv.vocab)

# 获得模型中词表的词数

print(len(model.wv.vocab))

# 获取模型中的语料数

print(model.corpus_count)

# 获取词向量的维度

print(model.wv.vector_size)

# # 获取某个词的词向量&＃xff0c; 先判断再获取

noun1 &＃61; &＃39;手机&＃39;

if noun1 in model.wv.vocab:

print(model.wv[noun1])

# 计算两个词的相似度(余弦距离)&＃xff0c;结果越大越相似

noun2 &＃61; &＃39;电池&＃39;

noun3 &＃61; &＃39;电量&＃39;

noun4 &＃61; &＃39;续航&＃39;

print(model.wv.similarity(noun1, noun2))

print(model.wv.similarity(noun3, noun2))

print(model.wv.similarity(noun4, noun2))

# 计算两个词的距离, 结果越大越不相似, 1-similarity

print(model.wv.distance(noun1, noun2))

print(model.wv.distance(noun3, noun2))

print(model.wv.distance(noun4, noun2))

# 取给定词最相近的topn个词

print(model.wv.most_similar(noun1))

# 找出与其他词差异最大的词

print(model.wv.doesnt_match([noun1, noun2, noun3, noun4]))

# 增量训练, 增加预料在原来模型的基础上训练

new_sentences &＃61; [[&＃39;手机&＃39;, &＃39;拍照&＃39;, &＃39;效果&＃39;, &＃39;不错&＃39;], [&＃39;相机&＃39;, &＃39;美颜&＃39;, &＃39;效果&＃39;, &＃39;好&＃39;]]

model.build_vocab(sentences&＃61;new_sentences, update&＃61;True)

model.train(sentences&＃61;new_sentences, total_examples&＃61;model.corpus_count, epochs&＃61;model.iter)

model.save(model_path)