如何用PaddlePaddle实现机器翻译？

示例数据

因为完整的数据集数据量较大，为了验证训练流程，PaddlePaddle接口paddle.dataset.wmt14中默认提供了一个经过预处理的较小规模的数据集。

该数据集有193319条训练数据，6003条测试数据，词典长度为30000。因为数据规模限制，使用该数据集训练出来的模型效果无法保证。

模型配置说明

下面我们开始根据输入数据的形式配置模型。首先引入所需的库函数以及定义全局变量。

from __future__ import print_functionimport paddleimport paddle.fluid as fluidimport paddle.fluid.layers as pdimport osimport systry:    from paddle.fluid.contrib.trainer import *    from paddle.fluid.contrib.inferencer import *except ImportError:    print(        "In the fluid 1.0, the trainer and inferencer are moving to paddle.fluid.contrib",        file=sys.stderr)    from paddle.fluid.trainer import *    from paddle.fluid.inferencer import *
dict_size = 30000 # 字典维度source_dict_dim = target_dict_dim = dict_size # 源/目标语言字典维度hidden_dim = 32 # 编码器中的隐层大小word_dim = 16 # 词向量维度batch_size = 2 # batch 中的样本数max_length = 8 # 生成句子的最大长度topk_size = 50beam_size = 2 # 柱宽度is_sparse = Truedecoder_size = hidden_dim # 解码器中的隐层大小model_save_dir = "machine_translation.inference.model"

然后如下实现编码器框架：

def encoder(is_sparse):  # 定义源语言id序列的输入数据  src_word_id = pd.data(      name="src_word_id", shape=[1], dtype='int64', lod_level=1)  # 将上述编码映射到低维语言空间的词向量  src_embedding = pd.embedding(      input=src_word_id,      size=[dict_size, word_dim],      dtype='float32',      is_sparse=is_sparse,      param_attr=fluid.ParamAttr(name='vemb'))  # LSTM层：fc + dynamic_lstm  fc1 = pd.fc(input=src_embedding, size=hidden_dim * 4, act='tanh')  lstm_hidden0, lstm_0 = pd.dynamic_lstm(input=fc1, size=hidden_dim * 4)  # 取源语言序列编码后的最后一个状态  encoder_out = pd.sequence_last_step(input=lstm_hidden0)  return encoder_out

再实现训练模式下的解码器：

def train_decoder(context):    # 定义目标语言id序列的输入数据，并映射到低维语言空间的词向量    trg_language_word = pd.data(        name="target_language_word", shape=[1], dtype='int64', lod_level=1)    trg_embedding = pd.embedding(        input=trg_language_word,        size=[dict_size, word_dim],        dtype='float32',        is_sparse=is_sparse,        param_attr=fluid.ParamAttr(name='vemb'))
    rnn = pd.DynamicRNN()    with rnn.block(): # 使用 DynamicRNN 定义每一步的计算        # 获取当前步目标语言输入的词向量        current_word = rnn.step_input(trg_embedding)        # 获取隐层状态        pre_state = rnn.memory(init=context)        # 解码器计算单元：单层前馈网络        current_state = pd.fc(input=[current_word, pre_state],                              size=decoder_size,                              act='tanh')        # 计算归一化的单词预测概率        current_score = pd.fc(input=current_state,                              size=target_dict_dim,                              act='softmax')        # 更新RNN的隐层状态        rnn.update_memory(pre_state, current_state)        # 输出预测概率        rnn.output(current_score)
    return rnn()

实现推测模式下的解码器：

def decode(context):    init_state = context    # 定义解码过程循环计数变量    array_len = pd.fill_constant(shape=[1], dtype='int64', value=max_length)    counter = pd.zeros(shape=[1], dtype='int64', force_cpu=True)
    # 定义 tensor array 用以保存各个时间步的内容，并写入初始id，score和state    state_array = pd.create_array('float32')    pd.array_write(init_state, array=state_array, i=counter)
    ids_array = pd.create_array('int64')    scores_array = pd.create_array('float32')
    init_ids = pd.data(name="init_ids", shape=[1], dtype="int64", lod_level=2)    init_scores = pd.data(        name="init_scores", shape=[1], dtype="float32", lod_level=2)
    pd.array_write(init_ids, array=ids_array, i=counter)    pd.array_write(init_scores, array=scores_array, i=counter)
    # 定义循环终止条件变量    cOnd= pd.less_than(x=counter, y=array_len)    # 定义 while_op    while_op = pd.While(cOnd=cond)    with while_op.block(): # 定义每一步的计算        # 获取解码器在当前步的输入，包括上一步选择的id，对应的score和上一步的state        pre_ids = pd.array_read(array=ids_array, i=counter)        pre_state = pd.array_read(array=state_array, i=counter)        pre_score = pd.array_read(array=scores_array, i=counter)
        # 更新输入的state为上一步选择id对应的state        pre_state_expanded = pd.sequence_expand(pre_state, pre_score)        # 同训练模式下解码器中的计算逻辑，包括获取输入向量，解码器计算单元计算和        # 归一化单词预测概率的计算        pre_ids_emb = pd.embedding(            input=pre_ids,            size=[dict_size, word_dim],            dtype='float32',            is_sparse=is_sparse，  param_attr=fluid.ParamAttr(name='vemb'))

        current_state = pd.fc(input=[pre_state_expanded, pre_ids_emb],                              size=decoder_size,                              act='tanh')        current_state_with_lod = pd.lod_reset(x=current_state, y=pre_score)        current_score = pd.fc(input=current_state_with_lod,                              size=target_dict_dim,                              act='softmax')        topk_scores, topk_indices = pd.topk(current_score, k=beam_size)

        # 计算累计得分，进行beam search        accu_scores = pd.elementwise_add(            x=pd.log(topk_scores), y=pd.reshape(pre_score, shape=[-1]), axis=0)        selected_ids, selected_scores = pd.beam_search(            pre_ids,            pre_score,            topk_indices,            accu_scores,            beam_size,            end_id=10,            level=0)    with pd.Switch() as switch:            with switch.case(pd.is_empty(selected_ids)):                pd.fill_constant(                    shape=[1], value=0, dtype='bool', force_cpu=True, out=cond)            with switch.default():                pd.increment(x=counter, value=1, in_place=True)
                pd.array_write(current_state, array=state_array, i=counter)                pd.array_write(selected_ids, array=ids_array, i=counter)                pd.array_write(selected_scores, array=scores_array, i=counter)
                length_cOnd= pd.less_than(x=counter, y=array_len)                finish_cOnd= pd.logical_not(pd.is_empty(x=selected_ids))                pd.logical_and(x=length_cond, y=finish_cond, out=cond)
    translation_ids, translation_scores = pd.beam_search_decode(        ids=ids_array, scores=scores_array, beam_size=beam_size, end_id=10)
    return translation_ids, translation_scores

进而，我们定义一个train_program来使用inference_program计算出的结果，在标记数据的帮助下来计算误差。我们还定义了一个optimizer_func来定义优化器:

def train_program():    cOntext= encoder()    rnn_out = train_decoder(context)    label = pd.data(        name="target_language_next_word", shape=[1], dtype='int64', lod_level=1)    cost = pd.cross_entropy(input=rnn_out, label=label)    avg_cost = pd.mean(cost)    return avg_cost
def optimizer_func():    return fluid.optimizer.Adagrad(        learning_rate=1e-4,        regularization=fluid.regularizer.L2DecayRegularizer(            regularization_coeff=0.1))

训练模型

定义训练环境

定义您的训练环境，可以指定训练是发生在CPU还是GPU上。

if use_cuda and not fluid.core.is_compiled_with_cuda():  returnplace = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()

定义数据提供器

下一步是为训练和测试定义数据提供器。提供器读入一个大小为 BATCH_SIZE的数据。paddle.dataset.wmt.train 每次会在乱序化后提供一个大小为BATCH_SIZE的数据，乱序化的大小为缓存大小buf_size。

train_reader = paddle.batch(        paddle.reader.shuffle(            paddle.dataset.wmt14.train(dict_size), buf_size=1000),        batch_size=batch_size)

构造训练器(trainer)

训练器需要一个训练程序和一个训练优化函数。

trainer = Trainer(        train_func=train_program, place=place, optimizer_func=optimizer_func)

提供数据

feed_order用来定义每条产生的数据和paddle.layer.data之间的映射关系。比如，wmt14.train产生的第一列的数据对应的是src_word_id这个特征。

feed_order = [        'src_word_id', 'target_language_word', 'target_language_next_word'    ]

事件处理器

回调函数event_handler在一个之前定义好的事件发生后会被调用。例如，我们可以在每步训练结束后查看误差。

def event_handler(event):    if isinstance(event, EndStepEvent):        if event.step % 10 == 0:            print('pass_id=' + str(event.epoch) + ' batch=' + str(event.step))  if isinstance(event, EndEpochEvent):    trainer.save_params(model_save_dir)

开始训练

最后，我们传入训练循环数（num_epoch）和一些别的参数，调用 trainer.train 来开始训练

trainer = Trainer(        train_func=train_program, place=place, optimizer_func=optimizer_func)trainer.train(        reader=train_reader,        num_epochs=EPOCH_NUM,        event_handler=event_handler,        feed_order=feed_order)

应用模型

定义解码部分

使用上面定义的 encoder 和 decoder 函数来推测翻译后的对应id和分数。

cOntext= encoder()translation_ids, translation_scores = decode(context)

定义数据

我们先初始化id和分数来生成tensors来作为输入数据。在这个预测例子中，我们用 wmt14.test 数据中的第一个记录来做推测，最后我们用"源字典"和"目标字典"来列印对应的句子结果。

init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64')init_scores_data = np.array(    [1. for _ in range(batch_size)], dtype='float32')init_ids_data = init_ids_data.reshape((batch_size, 1))init_scores_data = init_scores_data.reshape((batch_size, 1))init_lod = [1] * batch_sizeinit_lod = [init_lod, init_lod]
init_ids = fluid.create_lod_tensor(init_ids_data, init_lod, place)init_scores = fluid.create_lod_tensor(init_scores_data, init_lod, place)
test_data = paddle.batch(    paddle.reader.shuffle(        paddle.dataset.wmt14.test(dict_size), buf_size=1000),    batch_size=batch_size)
feed_order = ['src_word_id']feed_list = [    framework.default_main_program().global_block().var(var_name)    for var_name in feed_order]feeder = fluid.DataFeeder(feed_list, place)
src_dict, trg_dict = paddle.dataset.wmt14.get_dict(dict_size)

测试

现在我们可以进行预测了。我们要在 feed_order 提供对应参数，放在 executor 上运行以取得id和分数结果：

for data in test_data():    feed_data = map(lambda x: [x[0]], data)    feed_dict = feeder.feed(feed_data)    feed_dict['init_ids'] = init_ids    feed_dict['init_scores'] = init_scores
    results = exe.run(        framework.default_main_program(),        feed=feed_dict,        fetch_list=[translation_ids, translation_scores],        return_numpy=False)
    result_ids = np.array(results[0])    result_ids_lod = results[0].lod()    result_scores = np.array(results[1])
    print("Original sentence:")    print(" ".join([src_dict[w] for w in feed_data[0][0][1:-1]]))    print("Translated score and sentence:")    for i in xrange(beam_size):        start_pos = result_ids_lod[1][i] + 1        end_pos = result_ids_lod[1][i+1]        print("%d\t%.4f\t%s\n" % (i+1, result_scores[end_pos-1],                " ".join([trg_dict[w] for w in result_ids[start_pos:end_pos]])))
    break

总结

端到端的神经网络机器翻译是近几年兴起的一种全新的机器翻译方法。在本文中，我们介绍了NMT中典型的“编码器-解码器”框架。由于NMT是一个典型的Seq2Seq（Sequence to Sequence，序列到序列）学习问题，因此，Seq2Seq中的query改写（query rewriting）、摘要、单轮对话等问题都可以用本教程的模型来解决。

参考文献

1. Koehn P. Statistical machine translation[M]. Cambridge University Press, 2009.

2. Cho K, Van Merriënboer B, Gulcehre C, et al. Learning phrase representations using RNN encoder-decoderfor statistical machine translation[C]//Proceedings of the 2014 Conference on Empirical Methods inNatural Language Processing (EMNLP), 2014: 1724-1734.

3.Chung J, Gulcehre C, Cho K H, et al. Empiricalevaluation of gated recurrent neural networks on sequence modeling[J]. arXiv preprint arXiv:1412.3555, 2014.

4.Bahdanau D, Cho K, Bengio Y. Neuralmachine translation by jointly learning to align and translate[C]//Proceedings of ICLR 2015, 2015.

5.Papineni K, Roukos S, Ward T, et al. BLEU:a method for automatic evaluation of machine translation[C]//Proceedings of the 40th annual meeting on association forcomputational linguistics. Association for Computational Linguistics, 2002:311-318.