Python torch 模块，div() 实例源码
我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用torch.div()。
def validate(models, dataset, arg, cuda=False):
criterion = nn.MSELoss()
losses = []
batcher = dataset.get_batcher(shuffle=True, augment=False)
for b, (x, y) in enumerate(batcher, 1):
x = V(th.from_numpy(x).float()).cuda()
y = V(th.from_numpy(y).float()).cuda()
# Ensemble average
logit = None
for model, _ in models:
model.eval()
logit = model(x) if logit is None else logit + model(x)
logit = th.div(logit, len(models))
loss = criterion(logit, y)
losses.append(loss.data[0])
return np.mean(losses)
def predict(models, dataset, arg, cuda=False):
prediction_file = open('save/predictions.txt', 'w')
batcher = dataset.get_batcher(shuffle=False, augment=False)
for b, (x, _) in enumerate(batcher, 1):
x = V(th.from_numpy(x).float()).cuda()
# Ensemble average
logit = None
for model, _ in models:
model.eval()
logit = model(x) if logit is None else logit + model(x)
logit = th.div(logit, len(models))
prediction = logit.cpu().data[0][0]
prediction_file.write('%s\n' % prediction)
if arg.verbose and b % 100 == 0:
print('[predict] [b]:%s - prediction: %s' % (b, prediction))
# prediction_file.close()
def bn_hat_z_layers(self, hat_z_layers, z_pre_layers):
# TODO: Calculate batchnorm using GPU Tensors.
assert len(hat_z_layers) == len(z_pre_layers)
hat_z_layers_normalized = []
for i, (hat_z, z_pre) in enumerate(zip(hat_z_layers, z_pre_layers)):
if self.use_cuda:
ones = Variable(torch.ones(z_pre.size()[0], 1).cuda())
else:
ones = Variable(torch.ones(z_pre.size()[0], 1))
mean = torch.mean(z_pre, 0)
noise_var = np.random.normal(loc=0.0, scale=1 - 1e-10, size=z_pre.size())
if self.use_cuda:
var = np.var(z_pre.data.cpu().numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1])
else:
var = np.var(z_pre.data.numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1])
var = Variable(torch.FloatTensor(var))
if self.use_cuda:
hat_z = hat_z.cpu()
ones = ones.cpu()
mean = mean.cpu()
hat_z_normalized = torch.div(hat_z - ones.mm(mean), ones.mm(torch.sqrt(var + 1e-10)))
if self.use_cuda:
hat_z_normalized = hat_z_normalized.cuda()
hat_z_layers_normalized.append(hat_z_normalized)
return hat_z_layers_normalized
def train(epoch):
for e_ in range(epoch):
if (e_ + 1) % 10 == 0:
adjust_learning_rate(optimizer, e_)
cnt = 0
loss = Variable(torch.Tensor([0]))
for i_q, i_k, i_v, i_cand, i_a in zip(train_q, train_key,train_value, train_cand, train_a):
cnt += 1
i_q = i_q.unsqueeze(0) # add dimension
probs = model.forward(i_q, i_k, i_v,i_cand)
i_a = Variable(i_a)
curr_loss = loss_function(probs, i_a)
loss = torch.add(loss, torch.div(curr_loss, config.batch_size))
# naive batch implemetation, the lr is divided by batch size
if cnt % config.batch_size == 0:
print "Training loss", loss.data.sum()
loss.backward()
optimizer.step()
loss = Variable(torch.Tensor([0]))
model.zero_grad()
if cnt % config.valid_every == 0:
print "Accuracy:",eval()
def train(epoch):
for e_ in range(epoch):
if (e_ + 1) % 10 == 0:
adjust_learning_rate(optimizer, e_)
cnt = 0
loss = Variable(torch.Tensor([0]))
for i_q, i_w, i_e_p, i_a in zip(train_q, train_w, train_e_p, train_a):
cnt += 1
i_q = i_q.unsqueeze(0) # add dimension
probs = model.forward(i_q, i_w, i_e_p)
i_a = Variable(i_a)
curr_loss = loss_function(probs, i_a)
loss = torch.add(loss, torch.div(curr_loss, config.batch_size))
# naive batch implemetation, the lr is divided by batch size
if cnt % config.batch_size == 0:
print "Training loss", loss.data.sum()
loss.backward()
optimizer.step()
loss = Variable(torch.Tensor([0]))
model.zero_grad()
if cnt % config.valid_every == 0:
print "Accuracy:",eval()
def updateOutput(self, input):
assert input.dim() == 2
input_size = input.size()
self._output = self._output or input.new()
self.norm = self.norm or input.new()
self.buffer = self.buffer or input.new()
self._output.resize_as_(input)
# specialization for the infinity norm
if self.p == float('inf'):
if not self._indices:
self._indices = torch.cuda.FloatTensor() if torch.typename(self.output) == 'torch.cuda.FloatTensor' \
else torch.LongTensor()
torch.abs(self.buffer, input)
torch.max(self.norm, self._indices, self.buffer, 1)
self.norm.add_(self.eps)
else:
self.normp = self.normp or input.new()
if self.p % 2 != 0:
torch.abs(self.buffer, input).pow_(self.p)
else:
torch.pow(self.buffer, input, self.p)
torch.sum(self.normp, self.buffer, 1).add_(self.eps)
torch.pow(self.norm, self.normp, 1./self.p)
torch.div(self._output, input, self.norm.view(-1, 1).expand_as(input))
self.output = self._output.view(input_size)
return self.output
def updateGradInput(self, input, gradOutput):
if not self.gradInput:
return
self._div = self._div or input.new()
self._output = self._output or self.output.new()
self._gradOutput = self._gradOutput or input.new()
self._expand3 = self._expand3 or input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * (w_j - x) x - w_j
---- = ---------------- = -------
dx 2 || w_j - x || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(0.0000001)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(self._div, gradOutput, self._output)
assert input.dim() == 2
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand3 = self._div.expand(batchSize, inputSize, outputSize)
if torch.typename(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand3).copy_(self._expand3)
self._repeat2.mul_(self._repeat)
else:
torch.mul(self._repeat2, self._repeat, self._expand3)
torch.sum(self.gradInput, self._repeat2, 2)
self.gradInput.resize_as_(input)
return self.gradInput
def forward(self, input1, input2, y):
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
_idx = input1.new().byte()
buffer = torch.mul(input1, input2)
torch.sum(buffer, 1, out=self.w1)
epsilon = 1e-12
torch.mul(input1, input1, out=buffer)
torch.sum(buffer, 1, out=self.w22).add_(epsilon)
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self._outputs, self.w22, out=self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(input2, input2, out=buffer)
torch.sum(buffer, 1, out=self.w32).add_(epsilon)
torch.div(self._outputs, self.w32, out=self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self.w1, self.w, out=self._outputs)
self._outputs = self._outputs.select(1, 0)
torch.eq(y, -1, out=_idx)
self._outputs[_idx] = self._outputs[_idx].add_(-self.margin).clamp_(min=0)
torch.eq(y, 1, out=_idx)
self._outputs[_idx] = self._outputs[_idx].mul_(-1).add_(1)
output = self._outputs.sum()
if self.size_average:
output = output / y.size(0)
self.save_for_backward(input1, input2, y)
return input1.new((output,))
def updateOutput(self, input):
assert input.dim() == 2
input_size = input.size()
if self._output is None:
self._output = input.new()
if self.norm is None:
self.norm = input.new()
if self.buffer is None:
self.buffer = input.new()
self._output.resize_as_(input)
# specialization for the infinity norm
if self.p == float('inf'):
if not self._indices:
self._indices = torch.cuda.FloatTensor() if torch.typename(self.output) == 'torch.cuda.FloatTensor' \
else torch.LongTensor()
torch.abs(input, out=self.buffer)
torch.max(self._indices, self.buffer, 1, out=self.norm)
self.norm.add_(self.eps)
else:
if self.normp is None:
self.normp = input.new()
if self.p % 2 != 0:
torch.abs(input, out=self.buffer).pow_(self.p)
else:
torch.pow(input, self.p, out=self.buffer)
torch.sum(self.buffer, 1, out=self.normp).add_(self.eps)
torch.pow(self.normp, 1. / self.p, out=self.norm)
torch.div(input, self.norm.view(-1, 1).expand_as(input), out=self._output)
self.output = self._output.view(input_size)
return self.output
def forward(self, input1, input2, y):
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
_idx = input1.new().byte()
buffer = torch.mul(input1, input2)
torch.sum(buffer, 1, out=self.w1, keepdim=True)
epsilon = 1e-12
torch.mul(input1, input1, out=buffer)
torch.sum(buffer, 1, out=self.w22, keepdim=True).add_(epsilon)
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self._outputs, self.w22, out=self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(input2, input2, out=buffer)
torch.sum(buffer, 1, out=self.w32, keepdim=True).add_(epsilon)
torch.div(self._outputs, self.w32, out=self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self.w1, self.w, out=self._outputs)
self._outputs = self._outputs.select(1, 0)
torch.eq(y, -1, out=_idx)
self._outputs[_idx] = self._outputs[_idx].add_(-self.margin).clamp_(min=0)
torch.eq(y, 1, out=_idx)
self._outputs[_idx] = self._outputs[_idx].mul_(-1).add_(1)
output = self._outputs.sum()
if self.size_average:
output = output / y.size(0)
self.save_for_backward(input1, input2, y)
return input1.new((output,))
def updateOutput(self, input):
assert input.dim() == 2
input_size = input.size()
if self._output is None:
self._output = input.new()
if self.norm is None:
self.norm = input.new()
if self.buffer is None:
self.buffer = input.new()
self._output.resize_as_(input)
# specialization for the infinity norm
if self.p == float('inf'):
if not self._indices:
self._indices = torch.cuda.FloatTensor() if torch.typename(self.output) == 'torch.cuda.FloatTensor' \
else torch.LongTensor()
torch.abs(input, out=self.buffer)
torch.max(self._indices, self.buffer, 1, out=self.norm, keepdim=True)
self.norm.add_(self.eps)
else:
if self.normp is None:
self.normp = input.new()
if self.p % 2 != 0:
torch.abs(input, out=self.buffer).pow_(self.p)
else:
torch.pow(input, self.p, out=self.buffer)
torch.sum(self.buffer, 1, out=self.normp, keepdim=True).add_(self.eps)
torch.pow(self.normp, 1. / self.p, out=self.norm)
torch.div(input, self.norm.view(-1, 1).expand_as(input), out=self._output)
self.output = self._output.view(input_size)
return self.output
def forward(self, dec_state, context, mask=None):
"""
:param dec_state: batch x dec_dim
:param context: batch x T x enc_dim
:return: Weighted context, batch x enc_dim
Alpha weights (viz), batch x T
"""
batch, source_l, enc_dim = context.size()
assert enc_dim == self.enc_dim
# W*s over the entire batch (batch, attn_dim)
dec_contrib = self.decoder_in(dec_state)
# W*h over the entire length & batch (batch, source_l, attn_dim)
enc_contribs = self.encoder_in(
context.view(-1, self.enc_dim)).view(batch, source_l, self.attn_dim)
# tanh( Wh*hj + Ws s_{i-1} ) (batch, source_l, dim)
pre_attn = F.tanh(enc_contribs + dec_contrib.unsqueeze(1).expand_as(enc_contribs))
# v^T*pre_attn for all batches/lengths (batch, source_l)
energy = self.att_linear(pre_attn.view(-1, self.attn_dim)).view(batch, source_l)
# Apply the mask. (Might be a better way to do this)
if mask is not None:
shift = energy.max(1)[0]
energy_exp = (energy - shift.expand_as(energy)).exp() * mask
alpha = torch.div(energy_exp, energy_exp.sum(1).expand_as(energy_exp))
else:
alpha = F.softmax(energy)
weighted_context = torch.bmm(alpha.unsqueeze(1), context).squeeze(1) # (batch, dim)
return weighted_context, alpha
def rotation_error(input, target):
x1 = torch.norm(input, dim=1)
x2 = torch.norm(target, dim=1)
x1 = torch.div(input, torch.stack((x1, x1, x1, x1), dim=1))
x2 = torch.div(target, torch.stack((x2, x2, x2, x2), dim=1))
d = torch.abs(torch.sum(x1 * x2, dim=1))
theta = 2 * torch.acos(d) * 180/math.pi
theta = torch.mean(theta)
return theta
def rotation_error(input, target):
"""Gets cosine distance between input and target """
x1 = torch.norm(input, dim=1)
x2 = torch.norm(target, dim=1)
x1 = torch.div(input, torch.stack((x1, x1, x1, x1), dim=1))
x2 = torch.div(target, torch.stack((x2, x2, x2, x2), dim=1))
d = torch.abs(torch.sum(x1 * x2, dim=1))
theta = 2 * torch.acos(d) * 180/math.pi
theta = torch.mean(theta)
return theta
def forward(self, inpt):
batch_size = self.batch_size
f0 = self.features(inpt[:, 0])
f0 = f0.view(batch_size, -1)
f1 = self.features(inpt[:, 1])
f1 = f1.view(batch_size, -1)
# f2 = self.features(inpt[:, 2])
# f2 = f2.view(batch_size, -1)
#
# f3 = self.features(inpt[:, 3])
# f3 = f3.view(batch_size, -1)
#
# f4 = self.features(inpt[:, 4])
# f4 = f4.view(batch_size, -1)
#
# f = torch.stack((f0, f1, f2, f3, f4), dim=0).view(self.seq_length, batch_size, -1)
f = torch.cat((f0, f1), dim=1)
# _, hn = self.rnn(f, self.hidden)
# hn = hn[self.gru_layer - 1].view(batch_size, -1)
# hn = self.relu(hn)
# hn = self.dropout(hn)
# hn = self.regressor(hn)
hn = self.regressor(f)
trans = self.trans_regressor(hn)
# trans_norm = torch.norm(trans, dim=1)
# trans = torch.div(trans, torch.cat((trans_norm, trans_norm, trans_norm), dim=1))
scale = self.scale_regressor(hn)
rotation = self.rotation_regressor(hn)
return trans, scale, rotation
def updateOutput(self, input):
assert input.dim() == 2
input_size = input.size()
if self._output is None:
self._output = input.new()
if self.norm is None:
self.norm = input.new()
if self.buffer is None:
self.buffer = input.new()
self._output.resize_as_(input)
# specialization for the infinity norm
if self.p == float('inf'):
if not self._indices:
self._indices = torch.cuda.FloatTensor() if torch.typename(self.output) == 'torch.cuda.FloatTensor' \
else torch.LongTensor()
torch.abs(input, out=self.buffer)
torch.max(self._indices, self.buffer, 1, out=self.norm, keepdim=True)
self.norm.add_(self.eps)
else:
if self.normp is None:
self.normp = input.new()
if self.p % 2 != 0:
torch.abs(input, out=self.buffer).pow_(self.p)
else:
torch.pow(input, self.p, out=self.buffer)
torch.sum(self.buffer, 1, out=self.normp, keepdim=True).add_(self.eps)
torch.pow(self.normp, 1. / self.p, out=self.norm)
torch.div(input, self.norm.view(-1, 1).expand_as(input), out=self._output)
self.output = self._output.view(input_size)
return self.output
def l2_norm(self,input):
input_size = input.size()
buffer = torch.pow(input, 2)
normp = torch.sum(buffer, 1).add_(1e-10)
norm = torch.sqrt(normp)
_output = torch.div(input, norm.view(-1, 1).expand_as(input))
output = _output.view(input_size)
return output
def l2_norm(self,input):
input_size = input.size()
buffer = torch.pow(input, 2)
normp = torch.sum(buffer, 1).add_(1e-10)
norm = torch.sqrt(normp)
_output = torch.div(input, norm.view(-1, 1).expand_as(input))
output = _output.view(input_size)
return output
def normalize_batch(batch):
# normalize using imagenet mean and std
mean = batch.data.new(batch.data.size())
std = batch.data.new(batch.data.size())
mean[:, 0, :, :] = 0.485
mean[:, 1, :, :] = 0.456
mean[:, 2, :, :] = 0.406
std[:, 0, :, :] = 0.229
std[:, 1, :, :] = 0.224
std[:, 2, :, :] = 0.225
batch = torch.div(batch, 255.0)
batch -= Variable(mean)
batch = batch / Variable(std)
return batch
def batch_norm_scattering(x, m,v):
m=m.expand_as(x)
v=v.expand_as(x)
x = torch.div(torch.add(x,-m),v)
return x
def forward(self, input):
x = input
if x.data.is_cuda and self.gpuDevice != 0:
x = x.cuda(self.gpuDevice)
#
if x.size()[-1] == 128:
x = self.resize2(self.resize1(x))
x = self.layer8(self.layer7(self.layer6(self.layer5(
self.layer4(self.layer3(self.layer2(self.layer1(x))))))))
x = self.layer13(self.layer12(
self.layer11(self.layer10(self.layer9(x)))))
x = self.layer14(x)
x = self.layer15(x)
x = self.layer16(x)
x = self.layer17(x)
x = self.layer18(x)
x = self.layer19(x)
x = self.layer21(x)
x = self.layer22(x)
x = x.view((-1, 736))
x_736 = x
x = self.layer25(x)
x_norm = torch.sqrt(torch.sum(x**2, 1) + 1e-6)
x = torch.div(x, x_norm.view(-1, 1).expand_as(x))
return (x, x_736)
def updateOutput(self, input):
assert input.dim() == 2
input_size = input.size()
if self._output is None:
self._output = input.new()
if self.norm is None:
self.norm = input.new()
if self.buffer is None:
self.buffer = input.new()
self._output.resize_as_(input)
# specialization for the infinity norm
if self.p == float('inf'):
if not self._indices:
self._indices = torch.cuda.FloatTensor() if torch.typename(self.output) == 'torch.cuda.FloatTensor' \
else torch.LongTensor()
torch.abs(input, out=self.buffer)
torch.max(self._indices, self.buffer, 1, out=self.norm, keepdim=True)
self.norm.add_(self.eps)
else:
if self.normp is None:
self.normp = input.new()
if self.p % 2 != 0:
torch.abs(input, out=self.buffer).pow_(self.p)
else:
torch.pow(input, self.p, out=self.buffer)
torch.sum(self.buffer, 1, out=self.normp, keepdim=True).add_(self.eps)
torch.pow(self.normp, 1. / self.p, out=self.norm)
torch.div(input, self.norm.view(-1, 1).expand_as(input), out=self._output)
self.output = self._output.view(input_size)
return self.output
def columnwise_cosine_similarity(matrix1, matrix2):
"""Return the columnwise cosine similarity from matrix1 and matrix2.
Expect tesor of dimension (batch_size, seq_len, hidden).
Return tensor of size (batch_size, seq_len) containing the cosine
similarities."""
assert matrix1.size() == matrix2.size(), 'matrix sizes do not match'
# -> (batch_size, seq_len, 1)
n_m1 = torch.norm(matrix1, 2, 2)
n_m2 = torch.norm(matrix2, 2, 2)
# -> (batch_size, seq_len, 1)
col_norm = torch.mul(n_m1, n_m2)
# -> (batch_size, seq_len, hidden)
colprod = torch.mul(matrix1, matrix2)
# -> (batch_size, seq_len, 1)
colsum = torch.sum(colprod, 2)
# -> (batch_size, seq_len, 1)
cosine_sim = torch.div(colsum, col_norm)
# -> (batch_size, seq_len)
cosine_sim = cosine_sim.squeeze()
return cosine_sim
def full_cosine_similarity(matrix1, matrix2):
"""
Expect 2 matrices P and Q of dimension (d, n1) and (d, n2) respectively.
Return a matrix A of dimension (n1, n2) with the result of comparing each
vector to one another. A[i, j] represents the cosine similarity between
vectors P[:, i] and Q[:, j].
"""
n1 = matrix1.size(1)
n2 = matrix2.size(1)
d = matrix1.size(0)
assert d == matrix2.size(0)
# -> (d, n1, 1)
t1 = matrix1.view(d, n1, 1)
# -> (d, n1, n2)
t1 = t1.repeat(1, 1, n2)
# -> (d, 1, n2)
t2 = matrix2.view(d, 1, n2)
# -> (d, n1, n2)
t2 = t2.repeat(1, n1, 1).contiguous()
t1_x_t2 = torch.mul(t1, t2) # (d, n1, n2)
dotprod = torch.sum(t1_x_t2, 0).squeeze() # (n1, n2)
norm1 = torch.norm(t1, 2, 0) # (n1, n2)
norm2 = torch.norm(t2, 2, 0) # (n1, n2)
col_norm = torch.mul(norm1, norm2).squeeze() # (n1, n2)
return torch.div(dotprod, col_norm) # (n1, n2)
def batch_full_cosine_similarity(tensor1, tensor2):
"""
Expect 2 tensors tensor1 and tensor2 of dimension
(batch_size, seq_len_p, hidden) and (batch_size, seq_len_q, hidden)
respectively.
Return a matrix A of dimension (batch_size, seq_len_p, seq_len_q) with the
result of comparing each matrix to one another. A[k, :, :] represents the
cosine similarity between matrices P[k, :, :] and Q[k, :, :]. Then
A_k[i, j] is a scalar representing the cosine similarity between vectors
P_k[i, :] and Q_k[j, :]
"""
batch_size = tensor1.size(0)
seq_len_p = tensor1.size(1)
seq_len_q = tensor2.size(1)
hidden = tensor1.size(2)
assert batch_size == tensor2.size(0)
assert hidden == tensor2.size(2)
# -> (batch_size, seq_len_p, 1, hidden)
t1 = tensor1.unsqueeze(2)
# -> (batch_size, seq_len_p, seq_len_q, hidden)
t1 = t1.repeat(1, 1, seq_len_q, 1)
# -> (batch_size, 1, seq_len_q, hidden)
t2 = tensor2.unsqueeze(1)
# -> (batch_size, seq_len_p, seq_len_q, hidden)
t2 = t2.repeat(1, seq_len_p, 1, 1)
# -> (batch_size, seq_len_p, seq_len_q, hidden)
t1_x_t2 = torch.mul(t1, t2)
# -> (batch_size, seq_len_p, seq_len_q)
dotprod = torch.sum(t1_x_t2, 3).squeeze(3)
# norm1, norm2 and col_norm have dim (batch_size, seq_len_p, seq_len_q)
norm1 = torch.norm(t1, 2, 3)
norm2 = torch.norm(t2, 2, 3)
col_norm = torch.mul(norm1, norm2).squeeze(3)
return torch.div(dotprod, col_norm) # (batch_size, seq_len_p, seq_len_q)
def l2norm(X):
"""L2-normalize columns of X
"""
norm = torch.pow(X, 2).sum(dim=1, keepdim=True).sqrt()
X = torch.div(X, norm)
return X
def forward(self, input_n, hidden, phi, nh):
hidden = torch.cat((hidden, input_n), 2)
# Aggregate reresentations
h_conv = torch.div(torch.bmm(phi, hidden), nh)
hidden = hidden.view(-1, self.hidden_size + self.input_size)
h_conv = h_conv.view(-1, self.hidden_size + self.input_size)
# h_conv has shape (batch_size, n, hidden_size + input_size)
m1 = (torch.mm(hidden, self.W1)
.view(self.batch_size, -1, self.hidden_size))
m2 = (torch.mm(h_conv, self.W2)
.view(self.batch_size, -1, self.hidden_size))
m3 = self.b.unsqueeze(0).unsqueeze(1).expand_as(m2)
hidden = torch.sigmoid(m1 + m2 + m3)
return hidden
def forward(self, input_n, hidden, phi, nh):
hidden = torch.cat((hidden, input_n), 2)
# Aggregate reresentations
h_conv = torch.div(torch.bmm(phi, hidden), nh)
hidden = hidden.view(-1, self.hidden_size + self.input_size)
h_conv = h_conv.view(-1, self.hidden_size + self.input_size)
# h_conv has shape (batch_size, n, hidden_size + input_size)
m1 = (torch.mm(hidden, self.W1)
.view(self.batch_size, -1, self.hidden_size))
m2 = (torch.mm(h_conv, self.W2)
.view(self.batch_size, -1, self.hidden_size))
m3 = self.b.unsqueeze(0).unsqueeze(1).expand_as(m2)
hidden = torch.sigmoid(m1 + m2 + m3)
return hidden
def forward(self, input_n, hidden, phi, nh):
hidden = torch.cat((hidden, input_n), 2)
# Aggregate reresentations
h_conv = torch.div(torch.bmm(phi, hidden), nh)
hidden = hidden.view(-1, self.hidden_size + self.input_size + 2)
h_conv = h_conv.view(-1, self.hidden_size + self.input_size + 2)
# h_conv has shape (batch_size, n, hidden_size + input_size)
m1 = (torch.mm(hidden, self.W1)
.view(self.batch_size, -1, self.hidden_size))
m2 = (torch.mm(h_conv, self.W2)
.view(self.batch_size, -1, self.hidden_size))
m3 = self.b.unsqueeze(0).unsqueeze(1).expand_as(m2)
hidden = torch.sigmoid(m1 + m2 + m3)
return hidden
def forward(self, input_n, hidden, phi, nh):
self.batch_size = input_n.size()[0]
hidden = torch.cat((hidden, input_n), 2)
# Aggregate reresentations
h_conv = torch.div(torch.bmm(phi, hidden), nh)
hidden = hidden.view(-1, self.hidden_size + self.input_size)
h_conv = h_conv.view(-1, self.hidden_size + self.input_size)
# h_conv has shape (batch_size, n, hidden_size + input_size)
m1 = (torch.mm(hidden, self.W1)
.view(self.batch_size, -1, self.hidden_size))
m2 = (torch.mm(h_conv, self.W2)
.view(self.batch_size, -1, self.hidden_size))
m3 = self.b.unsqueeze(0).unsqueeze(1).expand_as(m2)
hidden = torch.sigmoid(m1 + m2 + m3)
return hidden
def backward(self, grad_output):
input, _ = self.saved_tensors
intersect, union = self.intersect, self.union
target = self.target_
gt = torch.div(target, union)
IoU2 = intersect/(union*union)
pred = torch.mul(input[:, 1], IoU2)
dDice = torch.add(torch.mul(gt, 2), torch.mul(pred, -4))
grad_input = torch.cat((torch.mul(dDice, -grad_output[0]),
torch.mul(dDice, grad_output[0])), 0)
return grad_input , None
def forward(self, input1, input2, y):
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
buffer = input1.new()
_idx = self._new_idx(input1)
torch.mul(buffer, input1, input2)
torch.sum(self.w1, buffer, 1)
epsilon = 1e-12
torch.mul(buffer, input1, input1)
torch.sum(self.w22, buffer, 1).add_(epsilon)
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self.w22, self._outputs, self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(buffer, input2, input2)
torch.sum(self.w32, buffer, 1).add_(epsilon)
torch.div(self.w32, self._outputs, self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self._outputs, self.w1, self.w)
self._outputs = self._outputs.select(1, 0)
torch.eq(_idx, y, -1)
self._outputs[_idx] = self._outputs[_idx].add_(-self.margin).cmax_(0)
torch.eq(_idx, y, 1)
self._outputs[_idx] = self._outputs[_idx].mul_(-1).add_(1)
output = self._outputs.sum()
if self.size_average:
output = output / y.size(0)
self.save_for_backward(input1, input2, y)
return input1.new((output,))
def updateOutput(self, input, y):
input1, input2 = input[0], input[1]
# keep backward compatibility
if not self.buffer:
self.buffer = input1.new()
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
# comparison operators behave differently from cuda/c implementations
# TODO: verify name
if input1.type() == 'torch.cuda.FloatTensor':
self._idx = torch.cuda.ByteTensor()
else:
self._idx = torch.ByteTensor()
torch.mul(self.buffer, input1, input2)
torch.sum(self.w1, self.buffer, 1)
epsilon = 1e-12
torch.mul(self.buffer, input1, input1)
torch.sum(self.w22, self.buffer, 1).add_(epsilon)
# self._outputs is also used as a temporary buffer
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self.w22, self._outputs, self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(self.buffer, input2, input2)
torch.sum(self.w32, self.buffer, 1).add_(epsilon)
torch.div(self.w32, self._outputs, self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self._outputs, self.w1, self.w)
self._outputs = self._outputs.select(1, 0)
torch.eq(self._idx, y, -1)
self._outputs[self._idx] = self._outputs[self._idx].add_(-self.margin).cmax_(0)
torch.eq(self._idx, y, 1)
self._outputs[self._idx] = self._outputs[self._idx].mul_(-1).add_(1)
self.output = self._outputs.sum()
if self.sizeAverage:
self.output = self.output / y.size(0)
return self.output
def updateGradInput(self, input, gradOutput):
if not self.gradInput:
return
self._div = self._div or input.new()
self._output = self._output or self.output.new()
self._expand4 = self._expand4 or input.new()
self._gradOutput = self._gradOutput or input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * c_j * c_j * (w_j - x) c_j * c_j * (x - w_j)
---- = -------------------------- = ---------------------
dx 2 || c_j * (w_j - x) || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(1e-7)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(self._div, gradOutput, self._output)
if input.dim() == 1:
self._div.resize_(1, outputSize)
self._expand4 = self._div.expand_as(self.weight)
if torch.type(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
else:
self._repeat2.mul_(self._repeat, self._expand4)
self._repeat2.mul_(self.diagCov)
torch.sum(self.gradInput, self._repeat2, 1)
self.gradInput.resize_as_(input)
elif input.dim() == 2:
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand4 = self._div.expand(batchSize, inputSize, outputSize)
if input.type() == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
self._repeat2.mul_(self._repeat3)
else:
torch.mul(self._repeat2, self._repeat, self._expand4)
self._repeat2.mul_(self._expand3)
torch.sum(self.gradInput, self._repeat2, 2)
self.gradInput.resize_as_(input)
else:
raise RuntimeError("1D or 2D input expected")
return self.gradInput
def pre_train_r(self):
print('=' * 50)
if cfg.ref_pre_path:
print('Loading R_pre from %s' % cfg.ref_pre_path)
self.R.load_state_dict(torch.load(cfg.ref_pre_path))
return
# we first train the R? network with just self-regularization loss for 1,000 steps
print('pre-training the refiner network %d times...' % cfg.r_pretrain)
for index in range(cfg.r_pretrain):
syn_image_batch, _ = self.syn_train_loader.__iter__().next()
syn_image_batch = Variable(syn_image_batch).cuda(cfg.cuda_num)
self.R.train()
ref_image_batch = self.R(syn_image_batch)
r_loss = self.self_regularization_loss(ref_image_batch, syn_image_batch)
# r_loss = torch.div(r_loss, cfg.batch_size)
r_loss = torch.mul(r_loss, self.delta)
self.opt_R.zero_grad()
r_loss.backward()
self.opt_R.step()
# log every `log_interval` steps
if (index % cfg.r_pre_per == 0) or (index == cfg.r_pretrain - 1):
# figure_name = 'refined_image_batch_pre_train_step_{}.png'.format(index)
print('[%d/%d] (R)reg_loss: %.4f' % (index, cfg.r_pretrain, r_loss.data[0]))
syn_image_batch, _ = self.syn_train_loader.__iter__().next()
syn_image_batch = Variable(syn_image_batch, volatile=True).cuda(cfg.cuda_num)
real_image_batch, _ = self.real_loader.__iter__().next()
real_image_batch = Variable(real_image_batch, volatile=True)
self.R.eval()
ref_image_batch = self.R(syn_image_batch)
figure_path = os.path.join(cfg.train_res_path, 'refined_image_batch_pre_train_%d.png' % index)
generate_img_batch(syn_image_batch.data.cpu(), ref_image_batch.data.cpu(),
real_image_batch.data, figure_path)
self.R.train()
print('Save R_pre to models/R_pre.pkl')
torch.save(self.R.state_dict(), 'models/R_pre.pkl')
def pre_train_d(self):
print('=' * 50)
if cfg.disc_pre_path:
print('Loading D_pre from %s' % cfg.disc_pre_path)
self.D.load_state_dict(torch.load(cfg.disc_pre_path))
return
# and D? for 200 steps (one mini-batch for refined images, another for real)
print('pre-training the discriminator network %d times...' % cfg.r_pretrain)
self.D.train()
self.R.eval()
for index in range(cfg.d_pretrain):
real_image_batch, _ = self.real_loader.__iter__().next()
real_image_batch = Variable(real_image_batch).cuda(cfg.cuda_num)
syn_image_batch, _ = self.syn_train_loader.__iter__().next()
syn_image_batch = Variable(syn_image_batch).cuda(cfg.cuda_num)
assert real_image_batch.size(0) == syn_image_batch.size(0)
# ============ real image D ====================================================
# self.D.train()
d_real_pred = self.D(real_image_batch).view(-1, 2)
d_real_y = Variable(torch.zeros(d_real_pred.size(0)).type(torch.LongTensor)).cuda(cfg.cuda_num)
d_ref_y = Variable(torch.ones(d_real_pred.size(0)).type(torch.LongTensor)).cuda(cfg.cuda_num)
acc_real = calc_acc(d_real_pred, 'real')
d_loss_real = self.local_adversarial_loss(d_real_pred, d_real_y)
# d_loss_real = torch.div(d_loss_real, cfg.batch_size)
# ============ syn image D ====================================================
# self.R.eval()
ref_image_batch = self.R(syn_image_batch)
# self.D.train()
d_ref_pred = self.D(ref_image_batch).view(-1, 2)
acc_ref = calc_acc(d_ref_pred, 'refine')
d_loss_ref = self.local_adversarial_loss(d_ref_pred, d_ref_y)
# d_loss_ref = torch.div(d_loss_ref, cfg.batch_size)
d_loss = d_loss_real + d_loss_ref
self.opt_D.zero_grad()
d_loss.backward()
self.opt_D.step()
if (index % cfg.d_pre_per == 0) or (index == cfg.d_pretrain - 1):
print('[%d/%d] (D)d_loss:%f acc_real:%.2f%% acc_ref:%.2f%%'
% (index, cfg.d_pretrain, d_loss.data[0], acc_real, acc_ref))
print('Save D_pre to models/D_pre.pkl')
torch.save(self.D.state_dict(), 'models/D_pre.pkl')
def updateOutput(self, input, y):
input1, input2 = input[0], input[1]
# keep backward compatibility
if self.buffer is None:
self.buffer = input1.new()
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
# comparison operators behave differently from cuda/c implementations
# TODO: verify name
if input1.type() == 'torch.cuda.FloatTensor':
self._idx = torch.cuda.ByteTensor()
else:
self._idx = torch.ByteTensor()
torch.mul(input1, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w1)
epsilon = 1e-12
torch.mul(input1, input1, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w22).add_(epsilon)
# self._outputs is also used as a temporary buffer
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self._outputs, self.w22, out=self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(input2, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w32).add_(epsilon)
torch.div(self._outputs, self.w32, out=self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self.w1, self.w, out=self._outputs)
self._outputs = self._outputs.select(1, 0)
torch.eq(y, -1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].add_(-self.margin).clamp_(min=0)
torch.eq(y, 1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].mul_(-1).add_(1)
self.output = self._outputs.sum()
if self.sizeAverage:
self.output = self.output / y.size(0)
return self.output
def updateGradInput(self, input, gradOutput):
if self.gradInput is None:
return
if self._div is None:
self._div = input.new()
if self._output is None:
self._output = self.output.new()
if self._gradOutput is None:
self._gradOutput = input.new()
if self._expand3 is None:
self._expand3 = input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * (w_j - x) x - w_j
---- = ---------------- = -------
dx 2 || w_j - x || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(0.0000001)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(gradOutput, self._output, out=self._div)
assert input.dim() == 2
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand3 = self._div.expand(batchSize, inputSize, outputSize)
if torch.typename(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand3).copy_(self._expand3)
self._repeat2.mul_(self._repeat)
else:
torch.mul(self._repeat, self._expand3, out=self._repeat2)
torch.sum(self._repeat2, 2, out=self.gradInput)
self.gradInput.resize_as_(input)
return self.gradInput
def updateGradInput(self, input, gradOutput):
if self.gradInput is None:
return
if self._div is None:
self._div = input.new()
if self._output is None:
self._output = self.output.new()
if self._expand4 is None:
self._expand4 = input.new()
if self._gradOutput is None:
self._gradOutput = input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * c_j * c_j * (w_j - x) c_j * c_j * (x - w_j)
---- = -------------------------- = ---------------------
dx 2 || c_j * (w_j - x) || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(1e-7)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(gradOutput, self._output, out=self._div)
if input.dim() == 1:
self._div.resize_(1, outputSize)
self._expand4 = self._div.expand_as(self.weight)
if torch.type(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
else:
self._repeat2.mul_(self._repeat, self._expand4)
self._repeat2.mul_(self.diagCov)
torch.sum(self._repeat2, 1, out=self.gradInput)
self.gradInput.resize_as_(input)
elif input.dim() == 2:
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand4 = self._div.expand(batchSize, inputSize, outputSize)
if input.type() == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
self._repeat2.mul_(self._repeat3)
else:
torch.mul(self._repeat, self._expand4, out=self._repeat2)
self._repeat2.mul_(self._expand3)
torch.sum(self._repeat2, 2, out=self.gradInput)
self.gradInput.resize_as_(input)
else:
raise RuntimeError("1D or 2D input expected")
return self.gradInput
def updateOutput(self, input, y):
input1, input2 = input[0], input[1]
# keep backward compatibility
if self.buffer is None:
self.buffer = input1.new()
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
# comparison operators behave differently from cuda/c implementations
# TODO: verify name
if input1.type() == 'torch.cuda.FloatTensor':
self._idx = torch.cuda.ByteTensor()
else:
self._idx = torch.ByteTensor()
torch.mul(input1, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w1, keepdim=True)
epsilon = 1e-12
torch.mul(input1, input1, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w22, keepdim=True).add_(epsilon)
# self._outputs is also used as a temporary buffer
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self._outputs, self.w22, out=self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(input2, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w32, keepdim=True).add_(epsilon)
torch.div(self._outputs, self.w32, out=self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self.w1, self.w, out=self._outputs)
self._outputs = self._outputs.select(1, 0)
torch.eq(y, -1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].add_(-self.margin).clamp_(min=0)
torch.eq(y, 1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].mul_(-1).add_(1)
self.output = self._outputs.sum()
if self.sizeAverage:
self.output = self.output / y.size(0)
return self.output
def updateGradInput(self, input, gradOutput):
if self.gradInput is None:
return
if self._div is None:
self._div = input.new()
if self._output is None:
self._output = self.output.new()
if self._gradOutput is None:
self._gradOutput = input.new()
if self._expand3 is None:
self._expand3 = input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * (w_j - x) x - w_j
---- = ---------------- = -------
dx 2 || w_j - x || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(0.0000001)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(gradOutput, self._output, out=self._div)
assert input.dim() == 2
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand3 = self._div.expand(batchSize, inputSize, outputSize)
if torch.typename(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand3).copy_(self._expand3)
self._repeat2.mul_(self._repeat)
else:
torch.mul(self._repeat, self._expand3, out=self._repeat2)
torch.sum(self._repeat2, 2, True, out=self.gradInput)
self.gradInput.resize_as_(input)
return self.gradInput
def updateGradInput(self, input, gradOutput):
if self.gradInput is None:
return
if self._div is None:
self._div = input.new()
if self._output is None:
self._output = self.output.new()
if self._expand4 is None:
self._expand4 = input.new()
if self._gradOutput is None:
self._gradOutput = input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * c_j * c_j * (w_j - x) c_j * c_j * (x - w_j)
---- = -------------------------- = ---------------------
dx 2 || c_j * (w_j - x) || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(1e-7)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(gradOutput, self._output, out=self._div)
if input.dim() == 1:
self._div.resize_(1, outputSize)
self._expand4 = self._div.expand_as(self.weight)
if torch.type(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
else:
self._repeat2.mul_(self._repeat, self._expand4)
self._repeat2.mul_(self.diagCov)
torch.sum(self._repeat2, 1, True, out=self.gradInput)
self.gradInput.resize_as_(input)
elif input.dim() == 2:
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand4 = self._div.expand(batchSize, inputSize, outputSize)
if input.type() == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
self._repeat2.mul_(self._repeat3)
else:
torch.mul(self._repeat, self._expand4, out=self._repeat2)
self._repeat2.mul_(self._expand3)
torch.sum(self._repeat2, 2, True, out=self.gradInput)
self.gradInput.resize_as_(input)
else:
raise RuntimeError("1D or 2D input expected")
return self.gradInput
def forward(ctx, input1, input2, y, margin, size_average):
ctx.margin = margin
ctx.size_average = size_average
ctx.w1 = input1.new()
ctx.w22 = input1.new()
ctx.w = input1.new()
ctx.w32 = input1.new()
ctx._outputs = input1.new()
_idx = input1.new().byte()
buffer = torch.mul(input1, input2)
torch.sum(buffer, 1, out=ctx.w1, keepdim=True)
epsilon = 1e-12
torch.mul(input1, input1, out=buffer)
torch.sum(buffer, 1, out=ctx.w22, keepdim=True).add_(epsilon)
ctx._outputs.resize_as_(ctx.w22).fill_(1)
torch.div(ctx._outputs, ctx.w22, out=ctx.w22)
ctx.w.resize_as_(ctx.w22).copy_(ctx.w22)
torch.mul(input2, input2, out=buffer)
torch.sum(buffer, 1, out=ctx.w32, keepdim=True).add_(epsilon)
torch.div(ctx._outputs, ctx.w32, out=ctx.w32)
ctx.w.mul_(ctx.w32)
ctx.w.sqrt_()
torch.mul(ctx.w1, ctx.w, out=ctx._outputs)
ctx._outputs = ctx._outputs.select(1, 0)
torch.eq(y, -1, out=_idx)
ctx._outputs[_idx] = ctx._outputs[_idx].add_(-ctx.margin).clamp_(min=0)
torch.eq(y, 1, out=_idx)
ctx._outputs[_idx] = ctx._outputs[_idx].mul_(-1).add_(1)
output = ctx._outputs.sum()
if ctx.size_average:
output = output / y.size(0)
ctx.save_for_backward(input1, input2, y)
return input1.new((output,))
def updateOutput(self, input, y):
input1, input2 = input[0], input[1]
# keep backward compatibility
if self.buffer is None:
self.buffer = input1.new()
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
# comparison operators behave differently from cuda/c implementations
# TODO: verify name
if input1.type() == 'torch.cuda.FloatTensor':
self._idx = torch.cuda.ByteTensor()
else:
self._idx = torch.ByteTensor()
torch.mul(input1, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w1, keepdim=True)
epsilon = 1e-12
torch.mul(input1, input1, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w22, keepdim=True).add_(epsilon)
# self._outputs is also used as a temporary buffer
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self._outputs, self.w22, out=self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(input2, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w32, keepdim=True).add_(epsilon)
torch.div(self._outputs, self.w32, out=self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self.w1, self.w, out=self._outputs)
self._outputs = self._outputs.select(1, 0)
torch.eq(y, -1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].add_(-self.margin).clamp_(min=0)
torch.eq(y, 1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].mul_(-1).add_(1)
self.output = self._outputs.sum()
if self.sizeAverage:
self.output = self.output / y.size(0)
return self.output
def updateGradInput(self, input, gradOutput):
if self.gradInput is None:
return
if self._div is None:
self._div = input.new()
if self._output is None:
self._output = self.output.new()
if self._gradOutput is None:
self._gradOutput = input.new()
if self._expand3 is None:
self._expand3 = input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * (w_j - x) x - w_j
---- = ---------------- = -------
dx 2 || w_j - x || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(0.0000001)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(gradOutput, self._output, out=self._div)
assert input.dim() == 2
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand3 = self._div.expand(batchSize, inputSize, outputSize)
if torch.typename(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand3).copy_(self._expand3)
self._repeat2.mul_(self._repeat)
else:
torch.mul(self._repeat, self._expand3, out=self._repeat2)
torch.sum(self._repeat2, 2, True, out=self.gradInput)
self.gradInput.resize_as_(input)
return self.gradInput
def updateGradInput(self, input, gradOutput):
if self.gradInput is None:
return
if self._div is None:
self._div = input.new()
if self._output is None:
self._output = self.output.new()
if self._expand4 is None:
self._expand4 = input.new()
if self._gradOutput is None:
self._gradOutput = input.new()
if not self.fastBackward:
self.updateOutput(input)
inputSize, outputSize = self.weight.size(0), self.weight.size(1)
"""
dy_j -2 * c_j * c_j * (w_j - x) c_j * c_j * (x - w_j)
---- = -------------------------- = ---------------------
dx 2 || c_j * (w_j - x) || y_j
"""
# to prevent div by zero (NaN) bugs
self._output.resize_as_(self.output).copy_(self.output).add_(1e-7)
self._view(self._gradOutput, gradOutput, gradOutput.size())
torch.div(gradOutput, self._output, out=self._div)
if input.dim() == 1:
self._div.resize_(1, outputSize)
self._expand4 = self._div.expand_as(self.weight)
if torch.type(input) == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
else:
self._repeat2.mul_(self._repeat, self._expand4)
self._repeat2.mul_(self.diagCov)
torch.sum(self._repeat2, 1, True, out=self.gradInput)
self.gradInput.resize_as_(input)
elif input.dim() == 2:
batchSize = input.size(0)
self._div.resize_(batchSize, 1, outputSize)
self._expand4 = self._div.expand(batchSize, inputSize, outputSize)
if input.type() == 'torch.cuda.FloatTensor':
self._repeat2.resize_as_(self._expand4).copy_(self._expand4)
self._repeat2.mul_(self._repeat)
self._repeat2.mul_(self._repeat3)
else:
torch.mul(self._repeat, self._expand4, out=self._repeat2)
self._repeat2.mul_(self._expand3)
torch.sum(self._repeat2, 2, True, out=self.gradInput)
self.gradInput.resize_as_(input)
else:
raise RuntimeError("1D or 2D input expected")
return self.gradInput
def bn_hat_z_layers(self, hat_z_layers, z_pre_layers):
# TODO: Calculate batchnorm using GPU Tensors.
assert len(hat_z_layers) == len(z_pre_layers)
hat_z_layers_normalized = []
for i, (hat_z, z_pre) in enumerate(zip(hat_z_layers, z_pre_layers)):
if self.use_cuda:
ones = Variable(torch.ones(z_pre.size()[0], 1).cuda())
else:
ones = Variable(torch.ones(z_pre.size()[0], 1))
mean = torch.mean(z_pre, 0)
noise_var = np.random.normal(loc=0.0, scale=1 - 1e-10, size=z_pre.size())
if self.use_cuda:
var = np.var(z_pre.data.cpu().numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1])
else:
var = np.var(z_pre.data.numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1])
var = Variable(torch.FloatTensor(var))
if self.use_cuda:
hat_z = hat_z.cpu()
ones = ones.cpu()
mean = mean.cpu()
"""
print(z_pre.data.shape, mean.data.shape, ones.data.shape, hat_z.data.shape)
print("=========== ")
print(z_pre)
print(mean)
print(ones)
print(hat_z)
print("=========== ")
"""
#ones = ones.unsqueeze(1)
mean = mean.unsqueeze(0)
#print(z_pre.data.shape, mean.data.shape, ones.data.shape, hat_z.data.shape)
tempa = hat_z - ones.mm(mean)
tempb = ones.mm(torch.sqrt(var + 1e-10))
#hat_z_normalized = torch.div(hat_z - ones.mm(mean), ones.mm(torch.sqrt(var + 1e-10)))
hat_z_normalized = torch.div(tempa, tempb)
if self.use_cuda:
hat_z_normalized = hat_z_normalized.cuda()
hat_z_layers_normalized.append(hat_z_normalized)
return hat_z_layers_normalized
def reward(sample_solution, USE_CUDA=False):
"""
The reward for the sorting task is defined as the
length of the longest sorted consecutive subsequence.
Input sequences must all be the same length.
Example:
input | output
====================
[1 4 3 5 2] | [5 1 2 3 4]
The output gets a reward of 4/5, or 0.8
The range is [1/sourceL, 1]
Args:
sample_solution: list of len sourceL of [batch_size]
Tensors
Returns:
[batch_size] containing trajectory rewards
"""
batch_size = sample_solution[0].size(0)
sourceL = len(sample_solution)
longest = Variable(torch.ones(batch_size), requires_grad=False)
current = Variable(torch.ones(batch_size), requires_grad=False)
if USE_CUDA:
longest = longest.cuda()
current = current.cuda()
for i in range(1, sourceL):
# compare solution[i-1] < solution[i]
res = torch.lt(sample_solution[i-1], sample_solution[i])
# if res[i,j] == 1, increment length of current sorted subsequence
current += res.float()
# else, reset current to 1
current[torch.eq(res, 0)] = 1
#current[torch.eq(res, 0)] -= 1
# if, for any, current > longest, update longest
mask = torch.gt(current, longest)
longest[mask] = current[mask]
return -torch.div(longest, sourceL)
def forward(ctx, input1, input2, y, margin, size_average):
ctx.margin = margin
ctx.size_average = size_average
ctx.w1 = input1.new()
ctx.w22 = input1.new()
ctx.w = input1.new()
ctx.w32 = input1.new()
ctx._outputs = input1.new()
_idx = input1.new().byte()
buffer = torch.mul(input1, input2)
torch.sum(buffer, 1, out=ctx.w1, keepdim=True)
epsilon = 1e-12
torch.mul(input1, input1, out=buffer)
torch.sum(buffer, 1, out=ctx.w22, keepdim=True).add_(epsilon)
ctx._outputs.resize_as_(ctx.w22).fill_(1)
torch.div(ctx._outputs, ctx.w22, out=ctx.w22)
ctx.w.resize_as_(ctx.w22).copy_(ctx.w22)
torch.mul(input2, input2, out=buffer)
torch.sum(buffer, 1, out=ctx.w32, keepdim=True).add_(epsilon)
torch.div(ctx._outputs, ctx.w32, out=ctx.w32)
ctx.w.mul_(ctx.w32)
ctx.w.sqrt_()
torch.mul(ctx.w1, ctx.w, out=ctx._outputs)
ctx._outputs = ctx._outputs.select(1, 0)
torch.eq(y, -1, out=_idx)
ctx._outputs[_idx] = ctx._outputs[_idx].add_(-ctx.margin).clamp_(min=0)
torch.eq(y, 1, out=_idx)
ctx._outputs[_idx] = ctx._outputs[_idx].mul_(-1).add_(1)
output = ctx._outputs.sum()
if ctx.size_average:
output = output / y.size(0)
ctx.save_for_backward(input1, input2, y)
return input1.new((output,))
def updateOutput(self, input, y):
input1, input2 = input[0], input[1]
# keep backward compatibility
if self.buffer is None:
self.buffer = input1.new()
self.w1 = input1.new()
self.w22 = input1.new()
self.w = input1.new()
self.w32 = input1.new()
self._outputs = input1.new()
# comparison operators behave differently from cuda/c implementations
# TODO: verify name
if input1.type() == 'torch.cuda.FloatTensor':
self._idx = torch.cuda.ByteTensor()
else:
self._idx = torch.ByteTensor()
torch.mul(input1, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w1, keepdim=True)
epsilon = 1e-12
torch.mul(input1, input1, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w22, keepdim=True).add_(epsilon)
# self._outputs is also used as a temporary buffer
self._outputs.resize_as_(self.w22).fill_(1)
torch.div(self._outputs, self.w22, out=self.w22)
self.w.resize_as_(self.w22).copy_(self.w22)
torch.mul(input2, input2, out=self.buffer)
torch.sum(self.buffer, 1, out=self.w32, keepdim=True).add_(epsilon)
torch.div(self._outputs, self.w32, out=self.w32)
self.w.mul_(self.w32)
self.w.sqrt_()
torch.mul(self.w1, self.w, out=self._outputs)
self._outputs = self._outputs.select(1, 0)
torch.eq(y, -1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].add_(-self.margin).clamp_(min=0)
torch.eq(y, 1, out=self._idx)
self._outputs[self._idx] = self._outputs[self._idx].mul_(-1).add_(1)
self.output = self._outputs.sum()
if self.sizeAverage:
self.output = self.output / y.size(0)
return self.output