递归神经网络部分组件（七）

递归神经网络的基础知识参考&＃xff1a;
https://www.zybuluo.com/hanbingtao/note/626300

自然语言和自然场景解析

• 在自然语言处理任务中&＃xff0c;如果我们能够实现一个解析器&＃xff0c;将自然语言解析为语法树&＃xff0c;那么毫无疑问&＃xff0c;这将大大提升我们对自然语言的处理能力。而递归神经网络就能够完成句子的语法分析&＃xff0c;并产生一个语法解析树。
• 除了自然语言之外&＃xff0c;自然场景也具有可组合的性质。因此&＃xff0c;我们可以用类似的模型完成自然场景的解析

下面是代码实现&＃xff1a;
代码中涉及的公式推导参考上面的连接。

activator.py

#!/usr/bin/env python
# -*- coding: UTF-8 -*-import numpy as npclass ReluActivator(object):def forward(self, weighted_input):#return weighted_inputreturn max(0, weighted_input)def backward(self, output):return 1 if output > 0 else 0class IdentityActivator(object):def forward(self, weighted_input):return weighted_inputdef backward(self, output):return 1class SigmoidActivator(object):def forward(self, weighted_input):return 1.0 / (1.0 &＃43; np.exp(-weighted_input))def backward(self, output):return output * (1 - output)class TanhActivator(object):def forward(self, weighted_input):return 2.0 / (1.0 &＃43; np.exp(-2 * weighted_input)) - 1.0def backward(self, output):return 1 - output * output

递归神经网络部分组件实现部分&＃xff1a;

# -*- coding: UTF-8 -*-import numpy as np
from activators import IdentityActivator# 用它保存递归神经网络生成的整棵树,用来构造父和子的关系
class TreeNode(object):def __init__(self, data, children&＃61;[], children_data&＃61;[]):self.parent &＃61; Noneself.children &＃61; childrenself.children_data &＃61; children_dataself.data &＃61; datafor child in children: # 设定子节点的父类child.parent &＃61; self #print &＃39;here:&＃39;,child.parent# 递归神经网络实现
class RecursiveLayer(object):def __init__(self, node_width, child_count, activator, learning_rate):&＃39;&＃39;&＃39;递归神经网络构造函数node_width: 表示每个节点的向量的维度child_count: 每个父节点有几个子节点activator: 激活函数对象learning_rate: 梯度下降算法学习率&＃39;&＃39;&＃39;self.node_width &＃61; node_widthself.child_count &＃61; child_countself.activator &＃61; activatorself.learning_rate &＃61; learning_rate# 权重数组Wself.W &＃61; np.random.uniform(-1e-4, 1e-4,(node_width, node_width * child_count))# 偏置项bself.b &＃61; np.zeros((node_width, 1))# 递归神经网络生成的树的根节点self.root &＃61; Nonedef forward(self, *children): # 可变参数的用法参考https://blog.csdn.net/lilong117194/article/details/80091217&＃39;&＃39;&＃39;前向计算&＃39;&＃39;&＃39;children_data &＃61; self.concatenate(children)#print &＃39;children_data:&＃39;, children_dataparent_data &＃61; self.activator.forward(np.dot(self.W, children_data) &＃43; self.b)#print &＃39;parent_data:&＃39;,parent_dataprint &＃39;children:&＃39;,childrenself.root &＃61; TreeNode(parent_data, children, children_data)print &＃39;tt..&＃39;,self.rootprint &＃39;self.root.children:&＃39;,self.root.childrendef backward(self, parent_delta):&＃39;&＃39;&＃39;BPTS反向传播算法&＃39;&＃39;&＃39;self.calc_delta(parent_delta, self.root)self.W_grad, self.b_grad &＃61; self.calc_gradient(self.root)def update(self):&＃39;&＃39;&＃39;使用SGD算法更新权重&＃39;&＃39;&＃39;self.W -&＃61; self.learning_rate * self.W_gradself.b -&＃61; self.learning_rate * self.b_graddef reset_state(self):self.root &＃61; Nonedef concatenate(self, tree_nodes):&＃39;&＃39;&＃39;将各个树节点中的数据拼接成一个长向量&＃39;&＃39;&＃39;concat &＃61; np.zeros((0,1))for node in tree_nodes:concat &＃61; np.concatenate((concat, node.data))#print &＃39;concat:&＃39;,concatreturn concat# 这里是难点def calc_delta(self, parent_delta, parent):&＃39;&＃39;&＃39;计算每个节点的delta&＃39;&＃39;&＃39;parent.delta &＃61; parent_deltaif parent.children:# 根据式2计算每个子节点的delta(只是下一层的)children_delta &＃61; np.dot(self.W.T, parent_delta) * (self.activator.backward(parent.children_data))print &＃39;children_delta——>&＃39;,children_delta# slices &＃61; [(子节点编号&＃xff0c;子节点delta起始位置&＃xff0c;子节点delta结束位置)]slices &＃61; [(i, i * self.node_width, (i &＃43; 1) * self.node_width)for i in range(self.child_count)]print &＃39;slices:&＃39;,slices# 针对每个子节点&＃xff0c;递归调用calc_delta函数for s in slices:print &＃39;children_delta[s[1]:s[2]]:&＃39;,children_delta[s[1]:s[2]]print &＃39;parent.children[s[0]]):&＃39;,parent.children[s[0]]self.calc_delta(children_delta[s[1]:s[2]], parent.children[s[0]])def calc_gradient(self, parent):&＃39;&＃39;&＃39;计算每个节点权重的梯度&＃xff0c;并将它们求和&＃xff0c;得到最终的梯度&＃39;&＃39;&＃39;W_grad &＃61; np.zeros((self.node_width, self.node_width * self.child_count))b_grad &＃61; np.zeros((self.node_width, 1))if not parent.children:return W_grad, b_gradparent.W_grad &＃61; np.dot(parent.delta, parent.children_data.T)parent.b_grad &＃61; parent.deltaW_grad &＃43;&＃61; parent.W_gradb_grad &＃43;&＃61; parent.b_gradfor child in parent.children:W, b &＃61; self.calc_gradient(child)W_grad &＃43;&＃61; Wb_grad &＃43;&＃61; breturn W_grad, b_grad# 打印输出def dump(self, **kwArgs):print &＃39;root.data: %s&＃39; % self.root.dataprint &＃39;root.children_data: %s&＃39; % self.root.children_dataif kwArgs.has_key(&＃39;dump_grad&＃39;):print &＃39;W_grad: %s&＃39; % self.W_gradprint &＃39;b_grad: %s&＃39; % self.b_graddef data_set():children &＃61; [TreeNode(np.array([[1],[2]])),TreeNode(np.array([[3],[4]])),TreeNode(np.array([[5],[6]]))]d &＃61; np.array([[0.5],[0.8]])return children, ddef gradient_check():&＃39;&＃39;&＃39;梯度检查&＃39;&＃39;&＃39;# 设计一个误差函数&＃xff0c;取所有节点输出项之和error_function &＃61; lambda o: o.sum()rnn &＃61; RecursiveLayer(2, 2, IdentityActivator(), 1e-3)# 计算forward值x, d &＃61; data_set()rnn.forward(x[0], x[1])rnn.forward(rnn.root, x[2])# 求取sensitivity mapsensitivity_array &＃61; np.ones((rnn.node_width, 1),dtype&＃61;np.float64)# 计算梯度rnn.backward(sensitivity_array)# 检查梯度epsilon &＃61; 10e-4for i in range(rnn.W.shape[0]):for j in range(rnn.W.shape[1]):rnn.W[i,j] &＃43;&＃61; epsilonrnn.reset_state()rnn.forward(x[0], x[1])rnn.forward(rnn.root, x[2])err1 &＃61; error_function(rnn.root.data)rnn.W[i,j] -&＃61; 2*epsilonrnn.reset_state()rnn.forward(x[0], x[1])rnn.forward(rnn.root, x[2])err2 &＃61; error_function(rnn.root.data)expect_grad &＃61; (err1 - err2) / (2 * epsilon)rnn.W[i,j] &＃43;&＃61; epsilonprint &＃39;weights(%d,%d): expected - actural %.4e - %.4e&＃39; % (i, j, expect_grad, rnn.W_grad[i,j])return rnndef test():children, d &＃61; data_set()# node_width, child_count, activator, learning_raternn &＃61; RecursiveLayer(2, 2, IdentityActivator(), 1e-3)rnn.forward(children[0], children[1])rnn.dump() # 打印输出父子节点的关系和datarnn.forward(rnn.root, children[2])rnn.dump()rnn.backward(d)rnn.dump(dump_grad&＃61;&＃39;true&＃39;)return rnntest()

运行结果&＃xff1a;

children: (<__main__.TreeNode object at 0x000000000BA4BF60>, <__main__.TreeNode object at 0x000000000BA4BD68>)
tt.. <__main__.TreeNode object at 0x000000000BA5E400>
self.root.children: (<__main__.TreeNode object at 0x000000000BA4BF60>, <__main__.TreeNode object at 0x000000000BA4BD68>)
root.data: [[ 8.10365462e-05][ -1.20068464e-06]]
root.children_data: [[ 1.][ 2.][ 3.][ 4.]]
children: (<__main__.TreeNode object at 0x000000000BA5E400>, <__main__.TreeNode object at 0x000000000BA4BC88>)
tt.. <__main__.TreeNode object at 0x000000000BAA6358>
self.root.children: (<__main__.TreeNode object at 0x000000000BA5E400>, <__main__.TreeNode object at 0x000000000BA4BC88>)
root.data: [[ 8.67226403e-05][ -2.18991385e-04]]
root.children_data: [[ 8.10365462e-05][ -1.20068464e-06][ 5.00000000e&＃43;00][ 6.00000000e&＃43;00]]
children_delta——> [[ -3.40659777e-05][ 7.72187116e-05][ -2.12167087e-05][ -4.29089851e-06]]
slices: [(0, 0, 2), (1, 2, 4)]
children_delta[s[1]:s[2]]: [[ -3.40659777e-05][ 7.72187116e-05]]
parent.children[s[0]]): <__main__.TreeNode object at 0x000000000BA5E400>
children_delta——> [[ -2.03214060e-09][ 4.49859993e-09][ -1.02736169e-08][ 5.25062172e-09]]
slices: [(0, 0, 2), (1, 2, 4)]
children_delta[s[1]:s[2]]: [[ -2.03214060e-09][ 4.49859993e-09]]
parent.children[s[0]]): <__main__.TreeNode object at 0x000000000BA4BF60>
children_delta[s[1]:s[2]]: [[ -1.02736169e-08][ 5.25062172e-09]]
parent.children[s[0]]): <__main__.TreeNode object at 0x000000000BA4BD68>
children_delta[s[1]:s[2]]: [[ -2.12167087e-05][ -4.29089851e-06]]
parent.children[s[0]]): <__main__.TreeNode object at 0x000000000BA4BC88>
root.data: [[ 8.67226403e-05][ -2.18991385e-04]]
root.children_data: [[ 8.10365462e-05][ -1.20068464e-06][ 5.00000000e&＃43;00][ 6.00000000e&＃43;00]]
W_grad: [[ 6.45229542e-06 -6.87322977e-05 2.49989780e&＃43;00 2.99986374e&＃43;00][ 1.42047949e-04 1.53476876e-04 4.00023166e&＃43;00 4.80030887e&＃43;00]]
b_grad: [[ 0.49996593][ 0.80007722]]

代码中做了一些打印输出&＃xff0c;以便于理解代码。

这里要注意几点&＃xff1a;

• 由于权重是在所有层共享的&＃xff0c;所以和循环神经网络一样&＃xff0c;递归神经网络的最终的权重梯度是各个层权重梯度之和。
• 网络的树构建大概是这样的&＃xff1a;
  
  具体的实现流程还要看代码。
• 这里没有进行梯度检查&＃xff0c;但在上一篇lstm中实现过&＃xff0c;这里的原理都是一样的。
• 整个代码架构的搭建是难点

通过初步学习递归神经网络的实现和应用场景&＃xff0c;感觉递归神经网络很神奇&＃xff0c;期待进一步的深入学习。