NLP训练个model出来写诗

couplet_file &＃61;"couplet.txt"#对联couplets &＃61; []with open(couplet_file,&＃39;r&＃39;) as f: for line in f: try: content &＃61; line.replace(&＃39; &＃39;,&＃39;&＃39;) if &＃39;_&＃39; in content or &＃39;(&＃39; in content or &＃39;&＃xff08;&＃39; in content or &＃39;《&＃39; in content or &＃39;[&＃39; in content: continue if len(content) <5*3 or len(content) > 79*3: continue content &＃61; &＃39;[&＃39; &＃43; content &＃43; &＃39;]&＃39; # print chardet.detect(content) content &＃61; content.decode(&＃39;utf-8&＃39;) couplets.append(content) except Exception as e: pass# 按字数排序couplets &＃61; sorted(couplets,key&＃61;lambda line: len(line))print(&＃39;对联总数: %d&＃39;%(len(couplets)))# 统计每个字出现次数all_words &＃61; []for couplet in couplets: all_words &＃43;&＃61; [word for word in couplet]counter &＃61; collections.Counter(all_words)count_pairs &＃61; sorted(counter.items(), key&＃61;lambda x: -x[1])words, _ &＃61; zip(*count_pairs)words &＃61; words[:len(words)] &＃43; (&＃39; &＃39;,)# 每个字映射为一个数字IDword_num_map &＃61; dict(zip(words, range(len(words))))to_num &＃61; lambda word: word_num_map.get(word, len(words))couplets_vector &＃61; [ list(map(to_num, couplet)) for couplet in couplets]# 每次取64首对联进行训练, 此参数可以调整batch_size &＃61; 64n_chunk &＃61; len(couplets_vector) // batch_sizex_batches &＃61; []y_batches &＃61; []for i in range(n_chunk): start_index &＃61; i * batch_size#起始位置 end_index &＃61; start_index &＃43; batch_size#结束位置 batches &＃61; couplets_vector[start_index:end_index] length &＃61; max(map(len,batches))#每个batches中句子的最大长度 xdata &＃61; np.full((batch_size,length), word_num_map[&＃39; &＃39;], np.int32) for row in range(batch_size): xdata[row,:len(batches[row])] &＃61; batches[row] ydata &＃61; np.copy(xdata) ydata[:,:-1] &＃61; xdata[:,1:] x_batches.append(xdata) y_batches.append(ydata)

定义LSTM模型&＃xff08;定义cell为一个128维的ht的cell。并使用MultiRNNCell 定义为两层的LSTM&＃xff09;def neural_network(rnn_size&＃61;128, num_layers&＃61;2): cell &＃61; tf.nn.rnn_cell.BasicLSTMCell(rnn_size, state_is_tuple&＃61;True) cell &＃61; tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers, state_is_tuple&＃61;True) initial_state &＃61; cell.zero_state(batch_size, tf.float32) with tf.variable_scope(&＃39;rnnlm&＃39;): softmax_w &＃61; tf.get_variable("softmax_w", [rnn_size, len(words)&＃43;1]) softmax_b &＃61; tf.get_variable("softmax_b", [len(words)&＃43;1]) with tf.device("/cpu:0"): embedding &＃61; tf.get_variable("embedding", [len(words)&＃43;1, rnn_size]) inputs &＃61; tf.nn.embedding_lookup(embedding, input_data) outputs, last_state &＃61; tf.nn.dynamic_rnn(cell, inputs, initial_state&＃61;initial_state, scope&＃61;&＃39;rnnlm&＃39;) output &＃61; tf.reshape(outputs,[-1, rnn_size]) logits &＃61; tf.matmul(output, softmax_w) &＃43; softmax_b probs &＃61; tf.nn.softmax(logits) return logits, last_state, probs, cell, initial_state

def train_neural_network(): logits, last_state, _, _, _ &＃61; neural_network() targets &＃61; tf.reshape(output_targets, [-1]) loss &＃61; tf.contrib.legacy_seq2seq.sequence_loss_by_example([logits], [targets], [tf.ones_like(targets, dtype&＃61;tf.float32)], len(words)) cost &＃61; tf.reduce_mean(loss) learning_rate &＃61; tf.Variable(0.0, trainable&＃61;False) tvars &＃61; tf.trainable_variables() grads, _ &＃61; tf.clip_by_global_norm(tf.gradients(cost, tvars), 5) optimizer &＃61; tf.train.AdamOptimizer(learning_rate) train_op &＃61; optimizer.apply_gradients(zip(grads, tvars)) with tf.Session() as sess: sess.run(tf.initialize_all_variables()) saver &＃61; tf.train.Saver(tf.all_variables()) for epoch in range(100): sess.run(tf.assign(learning_rate, 0.01 * (0.97 ** epoch))) n &＃61; 0 for batche in range(n_chunk): train_loss, _ , _ &＃61; sess.run([cost, last_state, train_op], feed_dict&＃61;{input_data: x_batches[n], output_targets: y_batches[n]}) n &＃43;&＃61; 1 print(epoch, batche, train_loss) if epoch % 7 &＃61;&＃61; 0: saver.save(sess, &＃39;./couplet.module&＃39;, global_step&＃61;epoch)

煌煌道宫&＃xff0c;肃肃太清。礼光尊祖&＃xff0c;乐备充庭。罄竭诚至&＃xff0c;希夷降灵。云凝翠盖&＃xff0c;风焰红旌。众真以从&＃xff0c;九奏初迎。永惟休v&＃xff0c;是锡和平。
种瓜黄台下&＃xff0c;瓜熟子离离。一摘使瓜好&＃xff0c;再摘使瓜稀。三摘犹自可&＃xff0c;摘绝抱蔓归。

from __future__ import print_function
import numpy as np
import tensorflow as tfimport argparse
import time
import os,sys
from six.moves import cPicklefrom utils import TextLoader
from model import Modeldef main():parser &＃61; argparse.ArgumentParser()parser.add_argument(&＃39;--save_dir&＃39;, type&＃61;str, default&＃61;&＃39;save&＃39;,help&＃61;&＃39;directory to store checkpointed models&＃39;)parser.add_argument(&＃39;--rnn_size&＃39;, type&＃61;int, default&＃61;128,help&＃61;&＃39;size of RNN hidden state&＃39;)parser.add_argument(&＃39;--num_layers&＃39;, type&＃61;int, default&＃61;2,help&＃61;&＃39;number of layers in the RNN&＃39;)parser.add_argument(&＃39;--model&＃39;, type&＃61;str, default&＃61;&＃39;lstm&＃39;,help&＃61;&＃39;rnn, gru, or lstm&＃39;)parser.add_argument(&＃39;--batch_size&＃39;, type&＃61;int, default&＃61;64,help&＃61;&＃39;minibatch size&＃39;)parser.add_argument(&＃39;--num_epochs&＃39;, type&＃61;int, default&＃61;50,help&＃61;&＃39;number of epochs&＃39;)parser.add_argument(&＃39;--save_every&＃39;, type&＃61;int, default&＃61;1000,help&＃61;&＃39;save frequency&＃39;)parser.add_argument(&＃39;--grad_clip&＃39;, type&＃61;float, default&＃61;5.,help&＃61;&＃39;clip gradients at this value&＃39;)parser.add_argument(&＃39;--learning_rate&＃39;, type&＃61;float, default&＃61;0.002,help&＃61;&＃39;learning rate&＃39;)parser.add_argument(&＃39;--decay_rate&＃39;, type&＃61;float, default&＃61;0.97,help&＃61;&＃39;decay rate for rmsprop&＃39;)parser.add_argument(&＃39;--init_from&＃39;, type&＃61;str, default&＃61;None,help&＃61;"""continue training from saved model at this path. Path must contain files saved by previous training process:&＃39;config.pkl&＃39; : configuration;&＃39;chars_vocab.pkl&＃39; : vocabulary definitions;&＃39;iterations&＃39; : number of trained iterations;&＃39;losses-*&＃39; : train loss;&＃39;checkpoint&＃39; : paths to model file(s) (created by tf).Note: this file contains absolute paths, be careful when moving files around;&＃39;model.ckpt-*&＃39; : file(s) with model definition (created by tf)""")args &＃61; parser.parse_args()train(args)def train(args):data_loader &＃61; TextLoader(args.batch_size)args.vocab_size &＃61; data_loader.vocab_size# check compatibility if training is continued from previously saved modelif args.init_from is not None:# check if all necessary files existassert os.path.isdir(args.init_from)," %s must be a a path" % args.init_fromassert os.path.isfile(os.path.join(args.init_from,"config.pkl")),"config.pkl file does not exist in path %s"%args.init_fromassert os.path.isfile(os.path.join(args.init_from,"chars_vocab.pkl")),"chars_vocab.pkl.pkl file does not exist in path %s" % args.init_fromckpt &＃61; tf.train.get_checkpoint_state(args.init_from)assert ckpt,"No checkpoint found"assert ckpt.model_checkpoint_path,"No model path found in checkpoint"assert os.path.isfile(os.path.join(args.init_from,"iterations")),"iterations file does not exist in path %s " % args.init_from# open old config and check if models are compatiblewith open(os.path.join(args.init_from, &＃39;config.pkl&＃39;),&＃39;rb&＃39;) as f:saved_model_args &＃61; cPickle.load(f)need_be_same&＃61;["model","rnn_size","num_layers"]for checkme in need_be_same:assert vars(saved_model_args)[checkme]&＃61;&＃61;vars(args)[checkme],"Command line argument and saved model disagree on &＃39;%s&＃39; "%checkme# open saved vocab/dict and check if vocabs/dicts are compatiblewith open(os.path.join(args.init_from, &＃39;chars_vocab.pkl&＃39;),&＃39;rb&＃39;) as f:saved_chars, saved_vocab &＃61; cPickle.load(f)assert saved_chars&＃61;&＃61;data_loader.chars, "Data and loaded model disagree on character set!"assert saved_vocab&＃61;&＃61;data_loader.vocab, "Data and loaded model disagree on dictionary mappings!"with open(os.path.join(args.save_dir, &＃39;config.pkl&＃39;), &＃39;wb&＃39;) as f:cPickle.dump(args, f)with open(os.path.join(args.save_dir, &＃39;chars_vocab.pkl&＃39;), &＃39;wb&＃39;) as f:cPickle.dump((data_loader.chars, data_loader.vocab), f)model &＃61; Model(args)with tf.Session() as sess:tf.global_variables_initializer().run()saver &＃61; tf.train.Saver(tf.global_variables())iterations &＃61; 0# restore model and number of iterationsif args.init_from is not None:saver.restore(sess, ckpt.model_checkpoint_path)with open(os.path.join(args.save_dir, &＃39;iterations&＃39;),&＃39;rb&＃39;) as f:iterations &＃61; cPickle.load(f)losses &＃61; []for e in range(args.num_epochs):sess.run(tf.assign(model.lr, args.learning_rate * (args.decay_rate ** e)))data_loader.reset_batch_pointer()for b in range(data_loader.num_batches):iterations &＃43;&＃61; 1start &＃61; time.time() x, y &＃61; data_loader.next_batch() feed &＃61; {model.input_data: x, model.targets: y}train_loss, _ , _ &＃61; sess.run([model.cost, model.final_state, model.train_op], feed)end &＃61; time.time()sys.stdout.write(&＃39;\r&＃39;)info &＃61; "{}/{} (epoch {}), train_loss &＃61; {:.3f}, time/batch &＃61; {:.3f}" \.format(e * data_loader.num_batches &＃43; b,args.num_epochs * data_loader.num_batches,e, train_loss, end - start)sys.stdout.write(info)sys.stdout.flush()losses.append(train_loss)if (e * data_loader.num_batches &＃43; b) % args.save_every &＃61;&＃61; 0\or (e&＃61;&＃61;args.num_epochs-1 and b &＃61;&＃61; data_loader.num_batches-1): # save for the last resultcheckpoint_path &＃61; os.path.join(args.save_dir, &＃39;model.ckpt&＃39;)saver.save(sess, checkpoint_path, global_step &＃61; iterations)with open(os.path.join(args.save_dir,"iterations"),&＃39;wb&＃39;) as f:cPickle.dump(iterations,f)with open(os.path.join(args.save_dir,"losses-"&＃43;str(iterations)),&＃39;wb&＃39;) as f:cPickle.dump(losses,f)losses &＃61; []sys.stdout.write(&＃39;\n&＃39;)print("model saved to {}".format(checkpoint_path))sys.stdout.write(&＃39;\n&＃39;)if __name__ &＃61;&＃61; &＃39;__main__&＃39;:main()

#-*- coding:utf-8 -*-import tensorflow as tf
from tensorflow.contrib import rnn
from tensorflow.contrib import legacy_seq2seq
import numpy as npclass Model():def __init__(self, args,infer&＃61;False):self.args &＃61; argsif infer:args.batch_size &＃61; 1if args.model &＃61;&＃61; &＃39;rnn&＃39;:cell_fn &＃61; rnn.BasicRNNCellelif args.model &＃61;&＃61; &＃39;gru&＃39;:cell_fn &＃61; rnn.GRUCellelif args.model &＃61;&＃61; &＃39;lstm&＃39;:cell_fn &＃61; rnn.BasicLSTMCellelse:raise Exception("model type not supported: {}".format(args.model))cell &＃61; cell_fn(args.rnn_size,state_is_tuple&＃61;False)self.cell &＃61; cell &＃61; rnn.MultiRNNCell([cell] * args.num_layers,state_is_tuple&＃61;False)self.input_data &＃61; tf.placeholder(tf.int32, [args.batch_size, None])# the length of input sequence is variable.self.targets &＃61; tf.placeholder(tf.int32, [args.batch_size, None])self.initial_state &＃61; cell.zero_state(args.batch_size, tf.float32)with tf.variable_scope(&＃39;rnnlm&＃39;):softmax_w &＃61; tf.get_variable("softmax_w", [args.rnn_size, args.vocab_size])softmax_b &＃61; tf.get_variable("softmax_b", [args.vocab_size])with tf.device("/cpu:0"):embedding &＃61; tf.get_variable("embedding", [args.vocab_size, args.rnn_size])inputs &＃61; tf.nn.embedding_lookup(embedding, self.input_data)outputs, last_state &＃61; tf.nn.dynamic_rnn(cell,inputs,initial_state&＃61;self.initial_state,scope&＃61;&＃39;rnnlm&＃39;)output &＃61; tf.reshape(outputs,[-1, args.rnn_size])self.logits &＃61; tf.matmul(output, softmax_w) &＃43; softmax_bself.probs &＃61; tf.nn.softmax(self.logits)targets &＃61; tf.reshape(self.targets, [-1])loss &＃61; legacy_seq2seq.sequence_loss_by_example([self.logits],[targets],[tf.ones_like(targets,dtype&＃61;tf.float32)],args.vocab_size)self.cost &＃61; tf.reduce_mean(loss)self.final_state &＃61; last_stateself.lr &＃61; tf.Variable(0.0, trainable&＃61;False)tvars &＃61; tf.trainable_variables()grads, _ &＃61; tf.clip_by_global_norm(tf.gradients(self.cost, tvars),args.grad_clip)optimizer &＃61; tf.train.AdamOptimizer(self.lr)self.train_op &＃61; optimizer.apply_gradients(zip(grads, tvars))def sample(self, sess, chars, vocab, prime&＃61;u&＃39;&＃39;, sampling_type&＃61;1):def pick_char(weights):if sampling_type &＃61;&＃61; 0:sample &＃61; np.argmax(weights)else:t &＃61; np.cumsum(weights)s &＃61; np.sum(weights)sample &＃61; int(np.searchsorted(t, np.random.rand(1)*s))return chars[sample]for char in prime:if char not in vocab:return u"{} is not in charset!".format(char)if not prime:state &＃61; self.cell.zero_state(1, tf.float32).eval()prime &＃61; u&＃39;^&＃39;result &＃61; u&＃39;&＃39;x &＃61; np.array([list(map(vocab.get,prime))])[probs,state] &＃61; sess.run([self.probs,self.final_state],{self.input_data: x,self.initial_state: state})char &＃61; pick_char(probs[-1])while char !&＃61; u&＃39;$&＃39;:result &＃43;&＃61; charx &＃61; np.zeros((1,1))x[0,0] &＃61; vocab[char][probs,state] &＃61; sess.run([self.probs,self.final_state],{self.input_data: x,self.initial_state: state})char &＃61; pick_char(probs[-1])return resultelse:result &＃61; u&＃39;^&＃39;for prime_char in prime:result &＃43;&＃61; prime_charx &＃61; np.array([list(map(vocab.get,result))])state &＃61; self.cell.zero_state(1, tf.float32).eval()[probs,state] &＃61; sess.run([self.probs,self.final_state],{self.input_data: x,self.initial_state: state})char &＃61; pick_char(probs[-1])while char !&＃61; u&＃39;&＃xff0c;&＃39; and char !&＃61; u&＃39;。&＃39;:result &＃43;&＃61; charx &＃61; np.zeros((1,1))x[0,0] &＃61; vocab[char][probs,state] &＃61; sess.run([self.probs,self.final_state],{self.input_data: x,self.initial_state: state})char &＃61; pick_char(probs[-1])result &＃43;&＃61; charreturn result[1:]

#-*- coding:utf-8 -*-import codecs
import os
import collections
from six.moves import cPickle,reduce,map
import numpy as npBEGIN_CHAR &＃61; &＃39;^&＃39;
END_CHAR &＃61; &＃39;$&＃39;
UNKNOWN_CHAR &＃61; &＃39;*&＃39;
MAX_LENGTH &＃61; 100class TextLoader():def __init__(self, batch_size, max_vocabsize&＃61;3000, encoding&＃61;&＃39;utf-8&＃39;):self.batch_size &＃61; batch_sizeself.max_vocabsize &＃61; max_vocabsizeself.encoding &＃61; encodingdata_dir &＃61; &＃39;./data&＃39;input_file &＃61; os.path.join(data_dir, "shijing.txt")vocab_file &＃61; os.path.join(data_dir, "vocab.pkl")tensor_file &＃61; os.path.join(data_dir, "data.npy")if not (os.path.exists(vocab_file) and os.path.exists(tensor_file)):print("reading text file")self.preprocess(input_file, vocab_file, tensor_file)else:print("loading preprocessed files")self.load_preprocessed(vocab_file, tensor_file)self.create_batches()self.reset_batch_pointer()def preprocess(self, input_file, vocab_file, tensor_file):def handle_poem(line):line &＃61; line.replace(&＃39; &＃39;,&＃39;&＃39;)if len(line) >&＃61; MAX_LENGTH:index_end &＃61; line.rfind(u&＃39;。&＃39;,0,MAX_LENGTH)index_end &＃61; index_end if index_end > 0 else MAX_LENGTHline &＃61; line[:index_end&＃43;1]return BEGIN_CHAR&＃43;line&＃43;END_CHARwith codecs.open(input_file, "r", encoding&＃61;self.encoding) as f:lines &＃61; list(map(handle_poem,f.read().strip().split(&＃39;\n&＃39;)))counter &＃61; collections.Counter(reduce(lambda data,line: line&＃43;data,lines,&＃39;&＃39;))count_pairs &＃61; sorted(counter.items(), key&＃61;lambda x: -x[1])chars, _ &＃61; zip(*count_pairs)self.vocab_size &＃61; min(len(chars),self.max_vocabsize - 1) &＃43; 1self.chars &＃61; chars[:self.vocab_size-1] &＃43; (UNKNOWN_CHAR,)self.vocab &＃61; dict(zip(self.chars, range(len(self.chars))))unknown_char_int &＃61; self.vocab.get(UNKNOWN_CHAR)with open(vocab_file, &＃39;wb&＃39;) as f:cPickle.dump(self.chars, f)get_int &＃61; lambda char: self.vocab.get(char,unknown_char_int)lines &＃61; sorted(lines,key&＃61;lambda line: len(line))self.tensor &＃61; [ list(map(get_int,line)) for line in lines ]with open(tensor_file,&＃39;wb&＃39;) as f:cPickle.dump(self.tensor,f)def load_preprocessed(self, vocab_file, tensor_file):with open(vocab_file, &＃39;rb&＃39;) as f:self.chars &＃61; cPickle.load(f)with open(tensor_file,&＃39;rb&＃39;) as f:self.tensor &＃61; cPickle.load(f)self.vocab_size &＃61; len(self.chars)self.vocab &＃61; dict(zip(self.chars, range(len(self.chars))))def create_batches(self):self.num_batches &＃61; int(len(self.tensor) / self.batch_size)self.tensor &＃61; self.tensor[:self.num_batches * self.batch_size]unknown_char_int &＃61; self.vocab.get(UNKNOWN_CHAR)self.x_batches &＃61; []self.y_batches &＃61; []for i in range(self.num_batches):from_index &＃61; i * self.batch_sizeto_index &＃61; from_index &＃43; self.batch_sizebatches &＃61; self.tensor[from_index:to_index]seq_length &＃61; max(map(len,batches))xdata &＃61; np.full((self.batch_size,seq_length),unknown_char_int,np.int32)for row in range(self.batch_size):xdata[row,:len(batches[row])] &＃61; batches[row]ydata &＃61; np.copy(xdata)ydata[:,:-1] &＃61; xdata[:,1:]self.x_batches.append(xdata)self.y_batches.append(ydata)def next_batch(self):x, y &＃61; self.x_batches[self.pointer], self.y_batches[self.pointer]self.pointer &＃43;&＃61; 1return x, ydef reset_batch_pointer(self):self.pointer &＃61; 0

#-*- coding:utf-8 -*-from __future__ import print_function
import numpy as np
import tensorflow as tf
import argparse
import time
import os
from six.moves import cPicklefrom utils import TextLoader
from model import Modelfrom six import text_typedef main():parser &＃61; argparse.ArgumentParser()parser.add_argument(&＃39;--save_dir&＃39;, type&＃61;str, default&＃61;&＃39;save&＃39;,help&＃61;&＃39;model directory to store checkpointed models&＃39;)parser.add_argument(&＃39;--prime&＃39;, type&＃61;str, default&＃61;&＃39;&＃39;,help&＃61;u&＃39;输入指定文字生成藏头诗&＃39;)parser.add_argument(&＃39;--sample&＃39;, type&＃61;int, default&＃61;1,help&＃61;&＃39;0 to use max at each timestep, 1 to sample at each timestep&＃39;)args &＃61; parser.parse_args()sample(args)def sample(args):with open(os.path.join(args.save_dir, &＃39;config.pkl&＃39;), &＃39;rb&＃39;) as f:saved_args &＃61; cPickle.load(f)with open(os.path.join(args.save_dir, &＃39;chars_vocab.pkl&＃39;), &＃39;rb&＃39;) as f:chars, vocab &＃61; cPickle.load(f)model &＃61; Model(saved_args, True)with tf.Session() as sess:tf.global_variables_initializer().run()saver &＃61; tf.train.Saver(tf.global_variables())ckpt &＃61; tf.train.get_checkpoint_state(args.save_dir)if ckpt and ckpt.model_checkpoint_path:saver.restore(sess, ckpt.model_checkpoint_path)print(model.sample(sess, chars, vocab, args.prime.decode(&＃39;utf-8&＃39;,errors&＃61;&＃39;ignore&＃39;), args.sample))if __name__ &＃61;&＃61; &＃39;__main__&＃39;:main()