Python tensorflow 模块,add_to_collection() 实例源码
我们从Python开源项目中,提取了以下50个代码示例,用于说明如何使用tensorflow.add_to_collection()。
def get_weight_variable(shape, name=None, type='xavier_uniform', regularize=True, **kwargs):
initialise_from_constant = False
if type == 'xavier_uniform':
initial = xavier_initializer(uniform=True, dtype=tf.float32)
elif type == 'xavier_normal':
initial = xavier_initializer(uniform=False, dtype=tf.float32)
elif type == 'he_normal':
initial = variance_scaling_initializer(uniform=False, factor=2.0, mode='FAN_IN', dtype=tf.float32)
elif type == 'he_uniform':
initial = variance_scaling_initializer(uniform=True, factor=2.0, mode='FAN_IN', dtype=tf.float32)
elif type == 'caffe_uniform':
initial = variance_scaling_initializer(uniform=True, factor=1.0, mode='FAN_IN', dtype=tf.float32)
elif type == 'simple':
stddev = kwargs.get('stddev', 0.02)
initial = tf.truncated_normal(shape, stddev=stddev, dtype=tf.float32)
initialise_from_constant = True
elif type == 'bilinear':
weights = _bilinear_upsample_weights(shape)
initial = tf.constant(weights, shape=shape, dtype=tf.float32)
initialise_from_constant = True
else:
raise ValueError('Unknown initialisation requested: %s' % type)
if name is None: # This keeps to option open to use unnamed Variables
weight = tf.Variable(initial)
else:
if initialise_from_constant:
weight = tf.get_variable(name, initializer=initial)
else:
weight = tf.get_variable(name, shape=shape, initializer=initial)
if regularize:
tf.add_to_collection('weight_variables', weight)
return weight
def model(self, features, labels):
x = features["observation"]
x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu)
x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu)
actions = tf.one_hot(tf.reshape(features["action"],[-1]), depth=6, on_value=1.0, off_value=0.0, axis=1)
x = tf.concat(1, [tf.contrib.layers.flatten(x), actions])
x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu)
x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu)
logits = tf.contrib.layers.fully_connected(x, 1, activation_fn=None)
prediction = tf.sigmoid(logits, name="prediction")
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.expand_dims(labels, axis=1)),name="loss")
train_op = tf.contrib.layers.optimize_loss(
loss, tf.contrib.framework.get_global_step(), optimizer='Adam',
learning_rate=self.learning_rate)
tf.add_to_collection('prediction', prediction)
tf.add_to_collection('loss', loss)
return prediction, loss, train_op
def _get_summary(self):
# build the self.summ_op for tensorboard
# This function could be overwritten
if not self.flags.visualize or self.flags.visualize=='none':
return
summ_collection = "{} {} {} summaries".format(self.flags.comp, self.flags.sol, self.flags.run_name)
if len(tf.get_collection(tf.GraphKeys.SCALARS)):
self.scaler_op = tf.summary.merge(tf.get_collection(tf.GraphKeys.SCALARS))
if len(tf.get_collection(tf.GraphKeys.IMAGES)):
self.image_op = tf.summary.merge(tf.get_collection(tf.GraphKeys.IMAGES))
for i in tf.get_collection(tf.GraphKeys.SCALARS):
tf.add_to_collection(summ_collection, i)
for i in tf.get_collection(tf.GraphKeys.WEIGHTS):
tf.add_to_collection(summ_collection, i)
for i in tf.get_collection(tf.GraphKeys.FEATURE_MAPS):
tf.add_to_collection(summ_collection, i)
for i in tf.get_collection(tf.GraphKeys.IMAGES):
tf.add_to_collection(summ_collection, i)
for i in tf.get_collection(tf.GraphKeys.GRADIENTS):
tf.add_to_collection(summ_collection, i)
for i in tf.get_collection(tf.GraphKeys.EMBEDDINGS):
tf.add_to_collection(summ_collection, i)
self.summ_op = tf.summary.merge(tf.get_collection(summ_collection))
def _variable_with_weight_decay(name, shape, wd, initializer, trainable=True):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_device(name, shape, initializer, trainable)
if wd is not None and trainable:
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def highway(self, input_1, input_2, size_1, size_2, l2_penalty=1e-8, layer_size=1):
output = input_2
for idx in range(layer_size):
with tf.name_scope('output_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_2,size_1], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[size_1]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
output = tf.nn.relu(tf.nn.xw_plus_b(output,W,b))
with tf.name_scope('transform_lin_%d' % idx):
W = tf.Variable(tf.truncated_normal([size_1,size_1], stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[size_1]), name="b")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
transform_gate = tf.sigmoid(tf.nn.xw_plus_b(input_1,W,b))
carry_gate = tf.constant(1.0) - transform_gate
output = transform_gate * output + carry_gate * input_1
return output
def conv_block(self, input, out_size, layer, kernalsize=3, l2_penalty=1e-8, shortcut=False):
in_shape = input.get_shape().as_list()
if layer>0:
filter_shape = [kernalsize, 1, in_shape[3], out_size]
else:
filter_shape = [kernalsize, in_shape[2], 1, out_size]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W-%s" % layer)
b = tf.Variable(tf.constant(0.1, shape=[out_size]), name="b-%s" % layer)
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(W))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(b))
if layer>0:
conv = tf.nn.conv2d(input, W, strides=[1, 1, 1, 1], padding="SAME", name="conv-%s" % layer)
else:
conv = tf.nn.conv2d(input, W, strides=[1, 1, 1, 1], padding="VALID", name="conv-%s" % layer)
if shortcut:
shortshape = [1,1,in_shape[3], out_size]
Ws = tf.Variable(tf.truncated_normal(shortshape, stddev=0.05), name="Ws-%s" % layer)
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(Ws))
conv = conv + tf.nn.conv2d(input, Ws, strides=[1, 1, 1, 1], padding="SAME", name="conv-shortcut-%s" % layer)
h = tf.nn.bias_add(conv, b)
h2 = tf.nn.relu(tf.contrib.layers.batch_norm(h, center=True, scale=True, epsilon=1e-5, decay=0.9), name="relu-%s" % layer)
return h2
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, **unused_params):
"""Creates a logistic model.
Args:
model_input: 'batch' x 'num_features' matrix of input features.
vocab_size: The number of classes in the dataset.
Returns:
A dictionary with a tensor containing the probability predictions of the
model in the 'predictions' key. The dimensions of the tensor are
batch_size x num_classes."""
input_size = vocab_size
output_size = FLAGS.hidden_size
with tf.name_scope("rbm"):
self.weights = tf.Variable(tf.truncated_normal([input_size, output_size],
stddev=1.0 / math.sqrt(float(input_size))), name="weights")
self.v_bias = tf.Variable(tf.zeros([input_size]), name="v_bias")
self.h_bias = tf.Variable(tf.zeros([output_size]), name="h_bias")
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(self.weights))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(self.v_bias))
tf.add_to_collection(name=tf.GraphKeys.REGULARIZATION_LOSSES, value=l2_penalty*tf.nn.l2_loss(self.h_bias))
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def get_video_weights(video_id_batch):
video_id_to_index = tf.contrib.lookup.string_to_index_table_from_file(
vocabulary_file=FLAGS.sample_vocab_file, default_value=0)
indexes = video_id_to_index.lookup(video_id_batch)
weights, length = get_video_weights_array()
weights_input = tf.placeholder(tf.float32, shape=[length], name="sample_weights_input")
weights_tensor = tf.get_variable("sample_weights",
shape=[length],
trainable=False,
dtype=tf.float32,
initializer=tf.constant_initializer(weights))
weights_assignment = tf.assign(weights_tensor, weights_input)
tf.add_to_collection("weights_input", weights_input)
tf.add_to_collection("weights_assignment", weights_assignment)
video_weight_batch = tf.nn.embedding_lookup(weights_tensor, indexes)
return video_weight_batch
def model(self, features, labels):
x = features["observation"]
x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu)
x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu)
x = tf.contrib.layers.flatten(x)
x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu)
x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu)
logits = tf.contrib.layers.fully_connected(x, 1, activation_fn=None)
prediction = tf.sigmoid(logits)
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.expand_dims(labels, axis=1)))
train_op = tf.contrib.layers.optimize_loss(
loss, tf.contrib.framework.get_global_step(), optimizer='Adam',
learning_rate=0.01)
tf.add_to_collection('prediction', prediction)
tf.add_to_collection('loss', loss)
return prediction, loss, train_op
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd:
# weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def loss(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd:
# weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def loss(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
var = _variable_on_cpu(
name,
shape,
tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
if wd is not None and not tf.get_variable_scope().reuse:
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def _add_cross_entropy(labels, logits, pref):
"""Compute average cross entropy and add to loss collection.
Args:
labels: Single dimension labels from distorted_inputs() or inputs().
logits: Output map from inference().
pref: Either 'c' or 's', for contours or segments, respectively.
"""
with tf.variable_scope('{}_cross_entropy'.format(pref)) as scope:
class_prop = C_CLASS_PROP if pref == 'c' else S_CLASS_PROP
weight_per_label = tf.scalar_mul(class_prop, tf.cast(tf.equal(labels, 0),
tf.float32)) + \
tf.scalar_mul(1.0 - class_prop, tf.cast(tf.equal(labels, 1),
tf.float32))
cross_entropy = tf.losses.sparse_softmax_cross_entropy(
labels=tf.squeeze(labels, squeeze_dims=[3]), logits=logits)
cross_entropy_weighted = tf.multiply(weight_per_label, cross_entropy)
cross_entropy_mean = tf.reduce_mean(cross_entropy_weighted, name=scope.name)
tf.add_to_collection('losses', cross_entropy_mean)
def inference(cls, lowres_input, fullres_input, params,
is_training=False):
with tf.variable_scope('coefficients'):
bilateral_coeffs = cls._coefficients(lowres_input, params, is_training)
tf.add_to_collection('bilateral_coefficients', bilateral_coeffs)
with tf.variable_scope('guide'):
guide = cls._guide(fullres_input, params, is_training)
tf.add_to_collection('guide', guide)
with tf.variable_scope('output'):
output = cls._output(
fullres_input, guide, bilateral_coeffs)
tf.add_to_collection('output', output)
return output
def inference(cls, lowres_input, fullres_input, params,
is_training=False):
with tf.variable_scope('coefficients'):
bilateral_coeffs = cls._coefficients(lowres_input, params, is_training)
tf.add_to_collection('bilateral_coefficients', bilateral_coeffs)
with tf.variable_scope('multiscale'):
multiscale = cls._multiscale_input(fullres_input)
for m in multiscale:
tf.add_to_collection('multiscale', m)
with tf.variable_scope('guide'):
guide = cls._guide(multiscale, params, is_training)
for g in guide:
tf.add_to_collection('guide', g)
with tf.variable_scope('output'):
output = cls._output(multiscale, guide, bilateral_coeffs)
tf.add_to_collection('output', output)
return output
def _get_weight_variable(self, layer_name, name, shape, L2=1):
wname = '%s/%s:0'%(layer_name,name)
fanin, fanout = shape[-2:]
for dim in shape[:-2]:
fanin *= float(dim)
fanout *= float(dim)
sigma = self._xavi_norm(fanin, fanout)
if self.weights is None or wname not in self.weights:
w1 = tf.get_variable(name,initializer=tf.truncated_normal(shape = shape,
mean=0,stddev = sigma))
print('{:>23} {:>23}'.format(wname, 'randomly initialize'))
else:
w1 = tf.get_variable(name, shape = shape,
initializer=tf.constant_initializer(value=self.weights[wname],dtype=tf.float32))
self.loaded_weights[wname]=1
if wname != w1.name:
print(wname,w1.name)
assert False
tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, tf.nn.l2_loss(w1)*L2)
return w1
def l1_loss(tensor, weight=1.0, scope=None):
"""Define a L1Loss, useful for regularize, i.e. lasso.
Args:
tensor: tensor to regularize.
weight: scale the loss by this factor.
scope: Optional scope for op_scope.
Returns:
the L1 loss op.
"""
with tf.op_scope([tensor], scope, 'L1Loss'):
weight = tf.convert_to_tensor(weight,
dtype=tensor.dtype.base_dtype,
name='loss_weight')
loss = tf.mul(weight, tf.reduce_sum(tf.abs(tensor)), name='value')
tf.add_to_collection(LOSSES_COLLECTION, loss)
return loss
def l2_loss(tensor, weight=1.0, scope=None, normalize=False):
"""Define a L2Loss, useful for regularize, i.e. weight decay.
Args:
tensor: tensor to regularize.
weight: an optional weight to modulate the loss.
scope: Optional scope for op_scope.
Returns:
the L2 loss op.
"""
with tf.op_scope([tensor], scope, 'L2Loss'):
weight = tf.convert_to_tensor(weight,
dtype=tensor.dtype.base_dtype,
name='loss_weight')
if normalize:
loss = tf.sqrt( (tf.sqrt( tf.nn.l2_loss(tensor)) / tf.to_float(tf.size(tensor))) , name='value')
else:
loss = tf.mul(weight, tf.nn.l2_loss(tensor), name='value')
tf.add_to_collection(LOSSES_COLLECTION, loss)
return loss
def sparse_cross_entropy_loss(logits, labels,
weight=1.0, scope=None):
"""Define a Cross Entropy loss using sparse_softmax_cross_entropy_with_logits.
It can scale the loss by weight factor, and smooth the labels.
Args:
logits: [batch_size, num_classes] logits outputs of the network .
labels: [batch_size,] target labels.
weight: scale the loss by this factor.
scope: Optional scope for op_scope.
Returns:
A tensor with the softmax_cross_entropy loss.
"""
with tf.op_scope([logits, labels], scope, 'SparseCrossEntropyLoss'):
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits,labels,name='xentropy')
weight = tf.convert_to_tensor(weight,
dtype=logits.dtype.base_dtype,
name='loss_weight')
loss = tf.mul(weight, tf.reduce_mean(cross_entropy), name='value')
tf.add_to_collection(LOSSES_COLLECTION, loss)
return loss
def add_variable(var, restore=True):
"""Adds a variable to the MODEL_VARIABLES collection.
Optionally it will add the variable to the VARIABLES_TO_RESTORE collection.
Args:
var: a variable.
restore: whether the variable should be added to the
VARIABLES_TO_RESTORE collection.
"""
collections = [MODEL_VARIABLES]
if restore:
collections.append(VARIABLES_TO_RESTORE)
for collection in collections:
if var not in tf.get_collection(collection):
tf.add_to_collection(collection, var)
def make_moving_average(name, value, init, decay, log=True):
"""Creates an exp-moving average of `value` and an update op, which is added to UPDATE_OPS collection.
:param name: string, name of the created moving average tf.Variable
:param value: tf.Tensor, the value to be averaged
:param init: float, an initial value for the moving average
:param decay: float between 0 and 1, exponential decay of the moving average
:param log: bool, add a summary op if True
:return: tf.Tensor, the moving average
"""
var = tf.get_variable(name, shape=value.get_shape(),
initializer=tf.constant_initializer(init), trainable=False)
update = moving_averages.assign_moving_average(var, value, decay, zero_debias=False)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update)
if log:
tf.summary.scalar(name, var)
return var
def _variable_with_weight_decay(self, name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
dtype = tf.float32 #if FLAGS.use_fp16 else tf.float32
var = self._variable_on_cpu(
name,
shape,
tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def variable_with_weight_decay(name, shape, stddev, wd):
"""
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name -> name of the variable
shape -> list of ints
stddev -> standard deviation of a truncated Gaussian
wd -> add L2Loss weight decay multiplied by this float.
If None, weight decay is not added for this Variable.
Rtns:
var -> variable tensor
"""
dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
var = variable_on_cpu(name,shape,
tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var),wd,name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def loss(logits, label_batch):
"""
Add L2Loss to all the trainable variables.
Add summary for "Loss" and "Loss/avg".
Args:
logits -> logits from inference()
label_batch -> 1D tensor of [batch_size]
Rtns:
total_loss -> float tensor
"""
# Calculate the average cross entropy loss across the batch.
label_batch = tf.cast(label_batch,tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits,
label_batch,name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses',cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def tower_loss(name_scope, autoencoder, clips):
# calculate reconstruction loss
rec_loss = tf.reduce_mean(tf.abs(clips-autoencoder.rec_vid))
weight_decay_loss_list = tf.get_collection('losses', name_scope)
weight_decay_loss = 0.0
if len(weight_decay_loss_list) > 0:
weight_decay_loss = tf.add_n(weight_decay_loss_list)
tf.add_to_collection('losses', rec_loss)
losses = tf.get_collection('losses', name_scope)
# Calculate the total loss for the current tower.
total_loss = tf.add_n(losses, name='total_loss')
return total_loss, rec_loss, weight_decay_loss
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(
name,
shape,
tf.truncated_normal_initializer(stddev=stddev, dtype=tf.float32))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def _loss_shared(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(
name,
shape,
tf.truncated_normal_initializer(stddev=stddev, dtype=tf.float32))
if wd is not None:
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def dense(x, size, name, weight_init=None, bias_init=0, weight_loss_dict=None, reuse=None):
with tf.variable_scope(name, reuse=reuse):
assert (len(U.scope_name().split('/')) == 2)
w = tf.get_variable("w", [x.get_shape()[1], size], initializer=weight_init)
b = tf.get_variable("b", [size], initializer=tf.constant_initializer(bias_init))
weight_decay_fc = 3e-4
if weight_loss_dict is not None:
weight_decay = tf.multiply(tf.nn.l2_loss(w), weight_decay_fc, name='weight_decay_loss')
if weight_loss_dict is not None:
weight_loss_dict[w] = weight_decay_fc
weight_loss_dict[b] = 0.0
tf.add_to_collection(U.scope_name().split('/')[0] + '_' + 'losses', weight_decay)
return tf.nn.bias_add(tf.matmul(x, w), b)
def embedding(indices, n_vocabulary, n_out,
weight_init=glorot_uniform,
reference=None,
trainable=True,
scope=None, reuse=False, name='Embedding'):
if type(n_out) is int:
n_out = (n_out,)
with tf.variable_scope(scope, default_name=name, reuse=reuse) as scope:
if reference is None:
W = tf.get_variable('W', shape=(n_vocabulary,) + n_out,
initializer=weight_init,
trainable=trainable)
else:
W = tf.get_variable('W', shape=(n_vocabulary,) + n_out,
initializer=reference_initializer(reference),
trainable=trainable)
tf.add_to_collection(tf.GraphKeys.WEIGHTS, W)
y = tf.nn.embedding_lookup(W, indices)
return y
def loss(self, predictions, real_values):
"""Return the loss operation between predictions and real_values.
Add L2 weight decay term if any.
Args:
predictions: predicted values
real_values: real values
Returns:
Loss tensor of type float.
"""
with tf.variable_scope('loss'):
# 1/2n \sum^{n}_{i=i}{(x_i - x'_i)^2}
mse = tf.divide(
tf.reduce_mean(
tf.square(tf.subtract(predictions, real_values))),
2.,
name="mse")
tf.add_to_collection(LOSSES, mse)
# mse + weight_decay per layer
error = tf.add_n(tf.get_collection(LOSSES), name='total_loss')
return error
def loss(self, logits, labels):
"""Add L2Loss to all the trainable variables.
Args:
logits: Logits from get().
labels: Labels from train_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
with tf.variable_scope('loss'):
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(
cross_entropy, name='cross_entropy')
tf.add_to_collection(LOSSES, cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
error = tf.add_n(tf.get_collection(LOSSES), name='total_loss')
return error
def loss(self, logits, labels):
"""Add L2Loss to all the trainable variables.
Args:
logits: Logits from get().
labels: Labels from train_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
with tf.variable_scope('loss'):
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(
cross_entropy, name='cross_entropy')
tf.add_to_collection(LOSSES, cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
error = tf.add_n(tf.get_collection(LOSSES), name='total_loss')
return error
def loss(self, logits, labels):
"""Add L2Loss to all the trainable variables.
Args:
logits: Logits from get().
labels: Labels from train_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
with tf.variable_scope('loss'):
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(
cross_entropy, name='cross_entropy')
tf.add_to_collection(LOSSES, cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
error = tf.add_n(tf.get_collection(LOSSES), name='total_loss')
return error
def loss(self, logits, labels):
"""Add L2Loss to all the trainable variables.
Args:
logits: Logits from get().
labels: Labels from train_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
with tf.variable_scope('loss'):
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(
cross_entropy, name='cross_entropy')
tf.add_to_collection(LOSSES, cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
error = tf.add_n(tf.get_collection(LOSSES), name='total_loss')
return error
def loss(self, logits, labels):
"""Add L2Loss to all the trainable variables.
Args:
logits: Logits from get().
labels: Labels from train_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
with tf.variable_scope('loss'):
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(
cross_entropy, name='cross_entropy')
tf.add_to_collection(LOSSES, cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
error = tf.add_n(tf.get_collection(LOSSES), name='total_loss')
return error
def loss(self, logits, labels):
"""Add L2Loss to all the trainable variables.
Args:
logits: Logits from get().
labels: Labels from train_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
with tf.variable_scope('loss'):
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(
cross_entropy, name='cross_entropy')
tf.add_to_collection(LOSSES, cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
error = tf.add_n(tf.get_collection(LOSSES), name='total_loss')
return error
def get_conv_filter(self, params):
if params["name"]+"/weights" in self.modelDict:
init = tf.constant_initializer(value=self.modelDict[params["name"]+"/weights"], dtype=tf.float32)
var = tf.get_variable(name="weights", initializer=init, shape=params["shape"])
print "loaded " + params["name"]+"/weights"
else:
if params["std"]:
stddev = params["std"]
else:
fanIn = params["shape"][0]*params["shape"][1]*params["shape"][2]
stddev = (2/float(fanIn))**0.5
init = tf.truncated_normal(shape=params["shape"], stddev=stddev, seed=0)
var = tf.get_variable(name="weights", initializer=init)
print "generated " + params["name"] + "/weights"
if not tf.get_variable_scope().reuse:
weightDecay = tf.mul(tf.nn.l2_loss(var), self._wd,
name='weight_loss')
tf.add_to_collection('losses', weightDecay)
return var
def get_conv_filter(self, params):
if params["name"]+"/weights" in self.modelDict:
init = tf.constant_initializer(value=self.modelDict[params["name"]+"/weights"], dtype=tf.float32)
var = tf.get_variable(name="weights", initializer=init, shape=params["shape"])
print "loaded " + params["name"]+"/weights"
else:
if params["std"]:
stddev = params["std"]
else:
fanIn = params["shape"][0]*params["shape"][1]*params["shape"][2]
stddev = (2/float(fanIn))**0.5
init = tf.truncated_normal(shape=params["shape"], stddev=stddev, seed=0)
var = tf.get_variable(name="weights", initializer=init)
print "generated " + params["name"] + "/weights"
if not tf.get_variable_scope().reuse:
weightDecay = tf.mul(tf.nn.l2_loss(var), self._wd,
name='weight_loss')
tf.add_to_collection('losses', weightDecay)
return var
def get_collection_handle(self, names):
"""
Get collection handles, collection are saved via `tf.add_to_collection`.
Arguments:
names: list of names
Returns:
List of ngraph ops corresponding to the names
"""
if self._graph is None:
raise ValueError("self.graph is empty, import meta_graph first.")
with self._graph.as_default():
tf_nodes = [tf.get_collection(name)[0] for name in names]
ng_ops = [self.get_op_handle(tf_node) for tf_node in tf_nodes]
return ng_ops
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def loss(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
var = _variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def loss(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
# Calculate the average cross entropy loss across the batch.
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')