使用Pytorch简单实现混合密度网络(MixtureDensityNetwork,MDN)

本文主要参考自&＃xff1a;
https://github.com/sksq96/pytorch-mdn/blob/master/mdn.ipynb
https://blog.otoro.net/2015/11/24/mixture-density-networks-with-tensorflow/?tdsourcetag&＃61;s_pctim_aiomsg

引言

我们知道&＃xff0c;神经网络具有很强的拟合能力。比方说&＃xff0c;假设我们要拟合如下一个带噪声的函数&＃xff1a;$$y\&＃61;7.0\sin (0.75x)\&＃43;0.5x\&＃43;ϵ$$

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from torchsummary import summaryimport numpy as np
import matplotlib.pyplot as plt
n_samples &＃61; 1000
epsilon &＃61; torch.randn(n_samples)
x_data &＃61; torch.linspace(-10, 10, n_samples)
y_data &＃61; 7*np.sin(0.75*x_data) &＃43; 0.5*x_data &＃43; epsilon
y_data, x_data &＃61; y_data.view(-1, 1), x_data.view(-1, 1)
plt.figure(figsize&＃61;(8, 8))
plt.scatter(x_data, y_data, alpha&＃61;0.4)
plt.show()


拟合这个显然是很容易的&＃xff0c;随便构建一个由两个全连接层组成的网络就具有非线性的拟合能力&＃xff1a;

n_input &＃61; 1
n_hidden &＃61; 20
n_output &＃61; 1
model &＃61; nn.Sequential(nn.Linear(n_input, n_hidden),nn.Tanh(),nn.Linear(n_hidden, n_output))
loss_fn &＃61; nn.MSELoss()
optimizer &＃61; torch.optim.RMSprop(model.parameters())


然后编写代码进行训练&＃xff1a;

for epoch in range(3000):y_pred &＃61; model(x_data)loss &＃61; loss_fn(y_pred, y_data)optimizer.zero_grad()loss.backward()optimizer.step()if epoch % 1000 &＃61;&＃61; 0:print(loss.data.tolist())x_test &＃61; torch.linspace(-15, 15, n_samples).view(-1, 1)
y_pred &＃61; model(x_test).data
plt.figure(figsize&＃61;(8, 8))
plt.scatter(x_data, y_data, alpha&＃61;0.4)
plt.scatter(x_test, y_pred, alpha&＃61;0.4, color&＃61;&＃39;red&＃39;)
plt.show()


结果如下&＃xff1a;

可以看到&＃xff0c;给定一个输入x&＃xff0c;要求预测一个y的话&＃xff0c;这种一对一的建模能力神经网络是很擅长的。但是实际工程中我们也很容易遇到这种情况&＃xff0c;在这里&＃xff0c;我们将x与y调换&＃xff0c;模拟一对多的情形&＃xff1a;$$x\&＃61;7.0\sin (0.75y)\&＃43;0.5y\&＃43;\epsilon$$

y_data, x_data &＃61; x_data.view(-1, 1), y_data.view(-1, 1)


重新训练进行预测&＃xff0c;结果如下&＃xff1a;

可以看到网络在试图拟合同一x下各y的平均值。那么&＃xff0c;有没有一种网络能够去拟合这种一对多的情况呢&＃xff1f;

MDN

传统的神经网络&＃xff1a;对于单一输入x&＃xff0c;给出一个单一预测y
MDN&＃xff1a;对于单一输入x&＃xff0c;预测y的概率分布

具体来说&＃xff0c;对于输入x&＃xff0c;MDN的输出为服从混合高斯分布(Mixture Gaussian distributions)&＃xff0c;具体的输出值被建模为多个高斯随机值的和&＃xff0c;这几个高斯分布的均值和标准差是不同的。形式化地&＃xff0c;有&＃xff1a;P(Y&＃61;y∣X&＃61;x)&＃61;∑k&＃61;0K−1Πk(x)ϕ(y,μk(x),σk(x)),∑k&＃61;0K−1Πk(x)&＃61;1P(Y&＃61;y \mid X&＃61;x)&＃61;\sum_{k&＃61;0}^{K-1} \Pi_{k}(x) \phi\left(y, \mu_{k}(x), \sigma_{k}(x)\right), \sum_{k&＃61;0}^{K-1} \Pi_{k}(x)&＃61;1 P(Y&＃61;y∣X&＃61;x)&＃61;k&＃61;0∑K−1​Πk​(x)ϕ(y,μk​(x),σk​(x)),k&＃61;0∑K−1​Πk​(x)&＃61;1 需要注意的是&＃xff0c;这里各个高斯分布的参数Πk(x),μk(x),σk(x)\Pi_{k}(x), \mu_{k}(x), \sigma_{k}(x)Πk​(x),μk​(x),σk​(x)是取决于输入$x$的&＃xff0c;也就是要通过网络训练去预测得到。实际上&＃xff0c;这依然可以通过全连接层来搞定&＃xff0c;接下来我们去介绍怎么实现MDN。

实现

class MDN(nn.Module):def __init__(self, n_hidden, n_gaussians):super(MDN, self).__init__()self.z_h &＃61; nn.Sequential(nn.Linear(1, n_hidden),nn.Tanh())self.z_pi &＃61; nn.Linear(n_hidden, n_gaussians)self.z_mu &＃61; nn.Linear(n_hidden, n_gaussians)self.z_sigma &＃61; nn.Linear(n_hidden, n_gaussians)def forward(self, x):z_h &＃61; self.z_h(x)pi &＃61; F.softmax(self.z_pi(z_h), -1)mu &＃61; self.z_mu(z_h)sigma &＃61; torch.exp(self.z_sigma(z_h))return pi, mu, sigma


混个高斯分布由多少个子高斯分布构成属于超参数&＃xff0c;这里我们设计为5个&＃xff1a;

model &＃61; MDN(n_hidden&＃61;20, n_gaussians&＃61;5)


然后是损失函数的设计。由于输出本质上是概率分布&＃xff0c;因此不能采用诸如L1损失、L2损失的硬损失函数。这里我们采用了对数似然损失(和交叉熵类似)&＃xff1a;CostFunction⁡(y∣x)&＃61;−log⁡[∑kKΠk(x)ϕ(y,μ(x),σ(x))]\operatorname{CostFunction}(y \mid x)&＃61;-\log \left[\sum_{k}^{K} \Pi_{k}(x) \phi(y, \mu(x), \sigma(x))\right]CostFunction(y∣x)&＃61;−log[k∑K​Πk​(x)ϕ(y,μ(x),σ(x))]

def mdn_loss_fn(y, mu, sigma, pi):m &＃61; torch.distributions.Normal(loc&＃61;mu, scale&＃61;sigma)loss &＃61; torch.exp(m.log_prob(y))loss &＃61; torch.sum(loss * pi, dim&＃61;1)loss &＃61; -torch.log(loss)return torch.mean(loss)


接下来则是训练的过程&＃xff1a;

for epoch in range(10000):pi, mu, sigma &＃61; model(x_data)loss &＃61; mdn_loss_fn(y_data, mu, sigma, pi)optimizer.zero_grad()loss.backward()optimizer.step()if epoch % 1000 &＃61;&＃61; 0:print(loss.data.tolist())


最后是推理的过程。需要注意的是&＃xff0c;MDN学到的只是若干个高斯分布&＃xff1a;

pi, mu, sigma &＃61; model(x_test)


因此我们还要手动去从高斯分布中采样获得具体的值&＃xff1a;

k &＃61; torch.multinomial(pi, 1).view(-1)
y_pred &＃61; torch.normal(mu, sigma)[np.arange(n_samples), k].data
plt.figure(figsize&＃61;(8, 8))
plt.scatter(x_data, y_data, alpha&＃61;0.4)
plt.scatter(x_test, y_pred, alpha&＃61;0.4, color&＃61;&＃39;red&＃39;)
plt.show()


最后结果如下&＃xff1a;

完整代码

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as pltn_samples &＃61; 1000epsilon &＃61; torch.randn(n_samples)
x_data &＃61; torch.linspace(-10, 10, n_samples)
y_data &＃61; 7*np.sin(0.75*x_data) &＃43; 0.5*x_data &＃43; epsilon# y_data, x_data &＃61; y_data.view(-1, 1), x_data.view(-1, 1),
y_data, x_data &＃61; x_data.view(-1, 1), y_data.view(-1, 1)plt.figure(figsize&＃61;(8, 8))
plt.scatter(x_data, y_data, alpha&＃61;0.4)
plt.show()n_input &＃61; 1
n_hidden &＃61; 20
n_output &＃61; 1
model &＃61; nn.Sequential(nn.Linear(n_input, n_hidden),nn.Tanh(),nn.Linear(n_hidden, n_output))
loss_fn &＃61; nn.MSELoss()
optimizer &＃61; torch.optim.RMSprop(model.parameters())for epoch in range(3000):y_pred &＃61; model(x_data)loss &＃61; loss_fn(y_pred, y_data)optimizer.zero_grad()loss.backward()optimizer.step()if epoch % 1000 &＃61;&＃61; 0:print(loss.data.tolist())x_test &＃61; torch.linspace(-15, 15, n_samples).view(-1, 1)
y_pred &＃61; model(x_test).data
plt.figure(figsize&＃61;(8, 8))
plt.scatter(x_data, y_data, alpha&＃61;0.4)
plt.scatter(x_test, y_pred, alpha&＃61;0.4, color&＃61;&＃39;red&＃39;)
plt.show()class MDN(nn.Module):def __init__(self, n_hidden, n_gaussians):super(MDN, self).__init__()self.z_h &＃61; nn.Sequential(nn.Linear(1, n_hidden),nn.Tanh())self.z_pi &＃61; nn.Linear(n_hidden, n_gaussians)self.z_mu &＃61; nn.Linear(n_hidden, n_gaussians)self.z_sigma &＃61; nn.Linear(n_hidden, n_gaussians)def forward(self, x):z_h &＃61; self.z_h(x)pi &＃61; F.softmax(self.z_pi(z_h), -1)mu &＃61; self.z_mu(z_h)sigma &＃61; torch.exp(self.z_sigma(z_h))return pi, mu, sigmamodel &＃61; MDN(n_hidden&＃61;20, n_gaussians&＃61;5)
optimizer &＃61; torch.optim.Adam(model.parameters())
def mdn_loss_fn(y, mu, sigma, pi):m &＃61; torch.distributions.Normal(loc&＃61;mu, scale&＃61;sigma)loss &＃61; torch.exp(m.log_prob(y))loss &＃61; torch.sum(loss * pi, dim&＃61;1)loss &＃61; -torch.log(loss)return torch.mean(loss)for epoch in range(10000):pi, mu, sigma &＃61; model(x_data)loss &＃61; mdn_loss_fn(y_data, mu, sigma, pi)optimizer.zero_grad()loss.backward()optimizer.step()if epoch % 1000 &＃61;&＃61; 0:print(loss.data.tolist())pi, mu, sigma &＃61; model(x_test)k &＃61; torch.multinomial(pi, 1).view(-1)
y_pred &＃61; torch.normal(mu, sigma)[np.arange(n_samples), k].dataplt.figure(figsize&＃61;(8, 8))
plt.scatter(x_data, y_data, alpha&＃61;0.4)
plt.scatter(x_test, y_pred, alpha&＃61;0.4, color&＃61;&＃39;red&＃39;)
plt.show()