词向量计算文本相似度,通过词向量求文本相似度

基于单词向量计算文本相似度1 .测试数据：链接： https://pan.Baidu.com/s/1 fxjjcujamawtfsutg2c BWA

抽屉代码： f4vx

2 .实验代码： importmathimportosimportpandasaspdimportnumpyasnp #两个向量的余弦相似度defcos_Similarity(vec_dim， 计算vector_1，输入vector_2) : #个向量#两个向量的点积x=0I=0#vec_dim=len(vector_1) # 向量维度whileIvec_dim3360x=xvector_1[ 2两个向量的模I=0sq _1=0sq _2=0while ivec _ dim 3360 sq _1=sq _1vevector ] vector_2[I]I=I1result=float(x )/(math.sqrt ) sq_1) math.sqrrt 3360withopen(path，encoding='utf8 ' ) ] row=data[0].split (()0) #embedding的行数col=data[0].split ) ) (1) #embedding的列数dim=int ) ) col (I=1 . split (' ) word=item [0] embedding=item [ 1: ] embedding=list (map ) ) )。 embedding ) (item_list.append ) word ) embeddings.append ) embedding ) I=1#embeddings.append ) item_list ) dimdeffind_each(path ) :path_list=[] files_dir=OS.listdir ) path ) forfileinfiles _ dir 3360 file _ patth file ) ) path_list.append(file_path ) path path_2) : #单输入vec _ dim _1=get _ embedding s (path _1) embeddings_2， vec _ dim _2=get _ embedding s (path _2) #成语向量相似度矩阵ifvec _ dim _1==vec _ dim _ 23360 matrix=[ ] forem _1ineer forem_2inem beddings _ 23360 cos _ sim=cos _ similarity (vec _ dim _ 1，em_1，em _2) score.aaad 输入错误：打印(input error : thedimensionsaredifferent )返回矩阵#卷积层，卷积核敏感区为2*2，参数为、 输入词向量矩阵defcnn_folding(dict_vec ) :c=len ) dict_vec[1]#获取输入矩阵的水平长度r=len ) dict_vec )获取输入矩阵的垂直长度result=

python构造的二维列表 for i in range(r-1): # 通过循环实现整个矩阵的运算 for j in range(c-1): re = (dict_vec[i][j] + dict_vec[i][j+1] + dict_vec[i+1][j] + dict_vec[i+1][j+1])/4 # 实现卷积层的运算，这里卷积核默认是[[1,1],[1,1]] result[i][j] = re return result# 池化层，采用max-pooling方式实现池化，参数表示输入矩阵def cnn_pooling(dict_pooling): c = len(dict_pooling[1]) r = len(dict_pooling) result = [[0 for col in range(c - 1)] for row in range(r - 1)] # python构造的二维列表 for i in range(r - 1): for j in range(c - 1): re = max(dict_pooling[i][j], dict_pooling[i][j + 1], dict_pooling[i + 1][j], dict_pooling[i + 1][j + 1]) # max-pooling方法实现池化 result[i][j] = re return result# 实现卷积层和池化层的连接层def pooling_folding(matrix): res = [] data_list = matrix while 1: # 交替实现卷积层和池化层 c = len(data_list[0]) r = len(data_list) if c == 1 or r == 1: # 判定池化层跳出循环条件 for i in range(len(data_list)): for j in data_list[i]: res.append(j) break pool = cnn_pooling(data_list) # 实现池化层 if len(pool) == 1 or len(pool[1]) == 1: # 判定卷积层跳出循环的条件 data_list = pool for i in range(len(data_list)): for j in data_list[i]: res.append(j) break else: fold = cnn_folding(pool) # 实现卷积层 data_list = fold pool = [[0 for col in range(c - 1)] for row in range(r - 1)] fold = [[0 for col in range(c - 1)] for row in range(r - 1)] return resjd_path = r'D:\thesis\0811\jd_graph\graph_embeddings'user_path = r'D:\thesis\0811\user_graph\graph_embeddings'jd_em_paths = find_each(jd_path) # 得到目录下的user_em_paths = find_each(user_path)job_list = []sim_lists = []for jd_file in jd_em_paths: sim_dict = {} jd_file_name = os.path.basename(jd_file) jd_name = jd_file_name.split('.')[0] # jd的类型名称 job_list.append(jd_name) for user_file in user_em_paths: sim_matrix = get_sim_matrix(jd_file, user_file) # 行代表job的embedding，列代表user的embedding，值为两个embedding的相似度 sim_res = pooling_folding(sim_matrix) # 送入卷积、池化层，全连接 sim_score = sum(sim_res)/len(sim_res) # 求和平均 user_file_name = os.path.basename(user_file) user_name = user_file_name.split('.')[0] # user id sim_dict.update({user_name: sim_score}) # 或.update(b=2) # 每个岗位与各用户的相似度 sim_list = sorted(zip(sim_dict.values(), sim_dict.keys()), reverse=True) # 降序排列 sim_list = sim_list[:100] # 取前100个 sim_lists.append(sim_list)df = pd.DataFrame()df['jd_sub_type'] = job_listdf['sim_users'] = sim_listsdf.to_csv("../data/jd_user_sim_2.csv", encoding="utf8", index=None, header=True) # 写入文件，每个岗位与各用户的相似度# df = pd.read_csv("../data/jd_user_sim.csv", encoding='utf8', header=0) # 读取文件print('end') 3.说明

代码实现的是两个文件夹中，文本embedding两两之间的相似度。测试只提供了两个embedding，需要更改合适的路径运行。

参考：https://blog.csdn.net/Mr_carry/article/details/80996454（有核心代码的详解。）