中文自然语言处理基于LSTM+Seq2Seq制作中文聊天机器人

Seq2Seq 属于 Encoder-Decoder 结构。基本思想就是利用两个 RNN&＃xff0c;一个 RNN 作为 Encoder&＃xff0c;另一个 RNN 作为 Decoder。Encoder 负责将输入序列压缩成指定长度的向量&＃xff0c;这个向量就可以看成是这个序列的语义&＃xff0c;这个过程称为编码。Decoder负责告诉模型对应的输出是什么。两个RNN通过语义向量连接&＃xff0c;得以实现输入与输出长度不同。

from keras.models import Model
from keras.layers import Input, LSTM, Dense
import numpy as np
import pandas as pd# 定义模型超参数、迭代次数、语料路径
#Batch size 的大小
batch_size &＃61; 32
# 迭代次数epochs
epochs &＃61; 100
# 编码空间的维度Latent dimensionality
latent_dim &＃61; 256
# 要训练的样本数
num_samples &＃61; 5000
#设置语料的路径
data_path &＃61; &＃39;./中文bot数据.txt&＃39;# 把语料向量化
input_texts &＃61; []
target_texts &＃61; []
input_characters &＃61; set()
target_characters &＃61; set()with open(data_path, &＃39;r&＃39;, encoding&＃61;&＃39;utf-8&＃39;) as f:lines &＃61; f.read().split(&＃39;\n&＃39;)
for line in lines[: min(num_samples, len(lines))]:# print(line)input_text, target_text &＃61; line.split(&＃39;\t&＃39;)# We use "tab" as the "start sequence" character# for the targets, and "\n" as "end sequence" character.target_text &＃61; target_text[0:100]target_text &＃61; &＃39;\t&＃39; &＃43; target_text &＃43; &＃39;\n&＃39;input_texts.append(input_text)target_texts.append(target_text)for char in input_text:if char not in input_characters:input_characters.add(char)for char in target_text:if char not in target_characters:target_characters.add(char)input_characters &＃61; sorted(list(input_characters))
target_characters &＃61; sorted(list(target_characters))
num_encoder_tokens &＃61; len(input_characters)
num_decoder_tokens &＃61; len(target_characters)
max_encoder_seq_length &＃61; max([len(txt) for txt in input_texts])
max_decoder_seq_length &＃61; max([len(txt) for txt in target_texts])print(&＃39;Number of samples:&＃39;, len(input_texts))
print(&＃39;Number of unique input tokens:&＃39;, num_encoder_tokens)
print(&＃39;Number of unique output tokens:&＃39;, num_decoder_tokens)
print(&＃39;Max sequence length for inputs:&＃39;, max_encoder_seq_length)
print(&＃39;Max sequence length for outputs:&＃39;, max_decoder_seq_length)input_token_index &＃61; dict([(char, i) for i, char in enumerate(input_characters)])
target_token_index &＃61; dict([(char, i) for i, char in enumerate(target_characters)])encoder_input_data &＃61; np.zeros((len(input_texts), max_encoder_seq_length, num_encoder_tokens), dtype&＃61;&＃39;float32&＃39;)
decoder_input_data &＃61; np.zeros((len(input_texts), max_decoder_seq_length, num_decoder_tokens), dtype&＃61;&＃39;float32&＃39;)
decoder_target_data &＃61; np.zeros((len(input_texts), max_decoder_seq_length, num_decoder_tokens), dtype&＃61;&＃39;float32&＃39;)for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):for t, char in enumerate(input_text):encoder_input_data[i, t, input_token_index[char]] &＃61; 1.for t, char in enumerate(target_text):# decoder_target_data is ahead of decoder_input_data by one timestepdecoder_input_data[i, t, target_token_index[char]] &＃61; 1.if t > 0:# decoder_target_data will be ahead by one timestep# and will not include the start character.decoder_target_data[i, t - 1, target_token_index[char]] &＃61; 1.# LSTM_Seq2Seq 模型定义、训练和保存
encoder_inputs &＃61; Input(shape&＃61;(None, num_encoder_tokens))
encoder &＃61; LSTM(latent_dim, return_state&＃61;True)
encoder_outputs, state_h, state_c &＃61; encoder(encoder_inputs)
# 输出 &＃96;encoder_outputs&＃96;
encoder_states &＃61; [state_h, state_c]
# 状态 &＃96;encoder_states&＃96;
decoder_inputs &＃61; Input(shape&＃61;(None, num_decoder_tokens))
decoder_lstm &＃61; LSTM(latent_dim, return_sequences&＃61;True, return_state&＃61;True)
decoder_outputs, _, _ &＃61; decoder_lstm(decoder_inputs,initial_state&＃61;encoder_states)
decoder_dense &＃61; Dense(num_decoder_tokens, activation&＃61;&＃39;softmax&＃39;)
decoder_outputs &＃61; decoder_dense(decoder_outputs)
# 定义模型
model &＃61; Model([encoder_inputs, decoder_inputs], decoder_outputs)
# 训练
model.compile(optimizer&＃61;&＃39;rmsprop&＃39;, loss&＃61;&＃39;categorical_crossentropy&＃39;)
model.summary()
model.fit([encoder_input_data, decoder_input_data], decoder_target_data,batch_size&＃61;batch_size,epochs&＃61;epochs,validation_split&＃61;0.2)
# 保存模型
model.save(&＃39;s2s.h5&＃39;)# Seq2Seq 的 Encoder 操作
encoder_model &＃61; Model(encoder_inputs, encoder_states)decoder_state_input_h &＃61; Input(shape&＃61;(latent_dim,))
decoder_state_input_c &＃61; Input(shape&＃61;(latent_dim,))
decoder_states_inputs &＃61; [decoder_state_input_h, decoder_state_input_c]
decoder_outputs, state_h, state_c &＃61; decoder_lstm(decoder_inputs, initial_state&＃61;decoder_states_inputs)
decoder_states &＃61; [state_h, state_c]
decoder_outputs &＃61; decoder_dense(decoder_outputs)
decoder_model &＃61; Model([decoder_inputs] &＃43; decoder_states_inputs,[decoder_outputs] &＃43; decoder_states)# 把索引和分词转成序列
reverse_input_char_index &＃61; dict((i, char) for char, i in input_token_index.items())
reverse_target_char_index &＃61; dict((i, char) for char, i in target_token_index.items())# 定义预测函数&＃xff0c;先使用预模型预测&＃xff0c;然后编码成汉字结果
def decode_sequence(input_seq):# Encode the input as state vectors.states_value &＃61; encoder_model.predict(input_seq)#print(states_value)# Generate empty target sequence of length 1.target_seq &＃61; np.zeros((1, 1, num_decoder_tokens))# Populate the first character of target sequence with the start character.target_seq[0, 0, target_token_index[&＃39;\t&＃39;]] &＃61; 1.# Sampling loop for a batch of sequences# (to simplify, here we assume a batch of size 1).stop_condition &＃61; Falsedecoded_sentence &＃61; &＃39;&＃39;while not stop_condition:output_tokens, h, c &＃61; decoder_model.predict([target_seq] &＃43; states_value)# Sample a tokensampled_token_index &＃61; np.argmax(output_tokens[0, -1, :])sampled_char &＃61; reverse_target_char_index[sampled_token_index]decoded_sentence &＃43;&＃61; sampled_charif (sampled_char &＃61;&＃61; &＃39;\n&＃39; orlen(decoded_sentence) > max_decoder_seq_length):stop_condition &＃61; True# Update the target sequence (of length 1).target_seq &＃61; np.zeros((1, 1, num_decoder_tokens))target_seq[0, 0, sampled_token_index] &＃61; 1.# 更新状态states_value &＃61; [h, c]return decoded_sentence# 模型预测
def predict_ans(question):inseq &＃61; np.zeros((1, max_encoder_seq_length, num_encoder_tokens), dtype&＃61;&＃39;float16&＃39;)for t, char in enumerate(question):inseq[0, t, input_token_index[char]] &＃61; 1.decoded_sentence &＃61; decode_sequence(inseq)return decoded_sentenceprint(&＃39;Decoded sentence:&＃39;, predict_ans("你是ai?"))

原文&＃xff1a;
https://soyoger.blog.csdn.net/article/details/108729400