我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.reduce_mean()。
def calculate_loss_mix2(self, predictions, predictions_class, predictions_encoder, labels, **unused_params): with tf.name_scope("loss_mix2"): float_labels = tf.cast(labels, tf.float32) float_encoders = float_labels for i in range(FLAGS.encoder_layers): var_i = np.loadtxt(FLAGS.autoencoder_dir+'autoencoder_layer%d.model' % i) weight_i = tf.constant(var_i[:-1,:],dtype=tf.float32) bias_i = tf.reshape(tf.constant(var_i[-1,:],dtype=tf.float32),[-1]) float_encoders = tf.nn.xw_plus_b(float_encoders,weight_i,bias_i) if i<FLAGS.encoder_layers-1: float_encoders = tf.nn.relu(float_encoders) else: hidden_mean = tf.reduce_mean(float_encoders,axis=1,keep_dims=True) hidden_std = tf.sqrt(tf.reduce_mean(tf.square(float_encoders-hidden_mean),axis=1,keep_dims=True)) float_encoders = (float_encoders-hidden_mean)/(hidden_std+1e-6) #float_encoders = tf.nn.sigmoid(float_encoders) cross_entropy_encoder = 0.1*self.calculate_mseloss(predictions_encoder,float_encoders) cross_entropy_loss = self.calculate_loss(predictions,labels) return cross_entropy_encoder+cross_entropy_loss, float_encoders #return cross_entropy_encoder, float_encoders
def model(self, features, labels): x = features["observation"] x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) actions = tf.one_hot(tf.reshape(features["action"],[-1]), depth=6, on_value=1.0, off_value=0.0, axis=1) x = tf.concat(1, [tf.contrib.layers.flatten(x), actions]) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) logits = tf.contrib.layers.fully_connected(x, 1, activation_fn=None) prediction = tf.sigmoid(logits, name="prediction") loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.expand_dims(labels, axis=1)),name="loss") train_op = tf.contrib.layers.optimize_loss( loss, tf.contrib.framework.get_global_step(), optimizer='Adam', learning_rate=self.learning_rate) tf.add_to_collection('prediction', prediction) tf.add_to_collection('loss', loss) return prediction, loss, train_op
def triplet_loss(anchor, positive, negative, alpha): """Calculate the triplet loss according to the FaceNet paper Args: anchor: the embeddings for the anchor images. positive: the embeddings for the positive images. negative: the embeddings for the negative images. Returns: the triplet loss according to the FaceNet paper as a float tensor. """ with tf.variable_scope('triplet_loss'): pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, positive)), 1) neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, negative)), 1) basic_loss = tf.add(tf.subtract(pos_dist,neg_dist), alpha) loss = tf.reduce_mean(tf.maximum(basic_loss, 0.0), 0) return loss
def build_model(self): self.q = tf.placeholder(tf.float32, [self.reader.vocab_size], name="question") self.a = tf.placeholder(tf.float32, [self.reader.vocab_size], name="answer") self.build_encoder() self.build_decoder() # Kullback Leibler divergence self.e_loss = -0.5 * tf.reduce_sum(1 + self.log_sigma_sq - tf.square(self.mu) - tf.exp(self.log_sigma_sq)) # Log likelihood self.g_loss = tf.reduce_sum(tf.log(self.p_x_i)) self.loss = tf.reduce_mean(self.e_loss + self.g_loss) self.optim = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(-self.loss) _ = tf.scalar_summary("encoder loss", self.e_loss) _ = tf.scalar_summary("decoder loss", self.g_loss) _ = tf.scalar_summary("loss", self.loss)
def log_variable(variable, gradient=None): r''' We introduce a function for logging a tensor variable's current state. It logs scalar values for the mean, standard deviation, minimum and maximum. Furthermore it logs a histogram of its state and (if given) of an optimization gradient. ''' name = variable.name mean = tf.reduce_mean(variable) tf.summary.scalar(name='%s/mean' % name, tensor=mean) tf.summary.scalar(name='%s/sttdev' % name, tensor=tf.sqrt(tf.reduce_mean(tf.square(variable - mean)))) tf.summary.scalar(name='%s/max' % name, tensor=tf.reduce_max(variable)) tf.summary.scalar(name='%s/min' % name, tensor=tf.reduce_min(variable)) tf.summary.histogram(name=name, values=variable) if gradient is not None: if isinstance(gradient, tf.IndexedSlices): grad_values = gradient.values else: grad_values = gradient if grad_values is not None: tf.summary.histogram(name='%s/gradients' % name, values=grad_values)
def _activation_summary(self, x, layer_name): """Helper to create summaries for activations. Args: x: layer output tensor layer_name: name of the layer Returns: nothing """ with tf.variable_scope('activation_summary') as scope: tf.summary.histogram( 'activation_summary/'+layer_name, x) tf.summary.scalar( 'activation_summary/'+layer_name+'/sparsity', tf.nn.zero_fraction(x)) tf.summary.scalar( 'activation_summary/'+layer_name+'/average', tf.reduce_mean(x)) tf.summary.scalar( 'activation_summary/'+layer_name+'/max', tf.reduce_max(x)) tf.summary.scalar( 'activation_summary/'+layer_name+'/min', tf.reduce_min(x))
def calculate_loss_distill_boost(self, predictions, labels_distill, labels, **unused_params): with tf.name_scope("loss_distill_boost"): print("loss_distill_boost") epsilon = 10e-6 float_labels = tf.cast(labels, tf.float32) batch_size = tf.shape(float_labels)[0] float_labels_distill = tf.cast(labels_distill, tf.float32) error = tf.negative(float_labels * tf.log(float_labels_distill + epsilon) + ( 1 - float_labels) * tf.log(1 - float_labels_distill + epsilon)) error = tf.reduce_sum(error,axis=1,keep_dims=True) alpha = error / tf.reduce_sum(error) * tf.cast(batch_size,dtype=tf.float32) alpha = tf.clip_by_value(alpha, 0.5, 5) alpha = alpha / tf.reduce_sum(alpha) * tf.cast(batch_size,dtype=tf.float32) cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + ( 1 - float_labels) * tf.log(1 - predictions + epsilon) cross_entropy_loss = tf.negative(cross_entropy_loss * alpha) return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def calculate_loss_distill_relabel(self, predictions, labels_distill, labels, **unused_params): with tf.name_scope("loss_distill_relabel"): print("loss_distill_relabel") epsilon = 10e-6 float_labels = tf.cast(labels, tf.float32) sum_labels = tf.cast(tf.reduce_sum(float_labels),dtype=tf.int32) pos_distill, _ = tf.nn.top_k(tf.reshape(labels_distill,[-1]), k=sum_labels) labels_true = tf.ones(tf.shape(labels)) labels_false = tf.zeros(tf.shape(labels)) labels_add = tf.where(tf.greater_equal(labels_distill, pos_distill[-1]), labels_true, labels_false) print(labels_add.get_shape().as_list()) float_labels = float_labels+labels_add*(1.0-float_labels) cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + ( 1 - float_labels) * tf.log(1 - predictions + epsilon) cross_entropy_loss = tf.negative(cross_entropy_loss) return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def calculate_loss(self, predictions, labels, **unused_params): with tf.name_scope("loss_xent"): epsilon = 10e-6 vocab_size = predictions.get_shape().as_list()[1] float_labels = tf.cast(labels, tf.float32) cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + ( 1 - float_labels) * tf.log(1 - predictions + epsilon) cross_entropy_loss = tf.negative(cross_entropy_loss) neg_labels = 1 - float_labels predictions_pos = predictions*float_labels+10*neg_labels predictions_minpos = tf.reduce_min(predictions_pos,axis=1,keep_dims=True) predictions_neg = predictions*neg_labels-10*float_labels predictions_maxneg = tf.reduce_max(predictions_neg,axis=1,keep_dims=True) mask_1 = tf.cast(tf.greater_equal(predictions_neg, predictions_minpos),dtype=tf.float32) mask_2 = tf.cast(tf.less_equal(predictions_pos, predictions_maxneg),dtype=tf.float32) cross_entropy_loss = cross_entropy_loss*(mask_1+mask_2)*10 + cross_entropy_loss return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def calculate_loss(self, predictions, labels, **unused_params): bound = FLAGS.softmax_bound vocab_size_1 = bound with tf.name_scope("loss_softmax"): epsilon = 10e-8 float_labels = tf.cast(labels, tf.float32) labels_1 = float_labels[:,:vocab_size_1] predictions_1 = predictions[:,:vocab_size_1] cross_entropy_loss = CrossEntropyLoss().calculate_loss(predictions_1,labels_1) lables_2 = float_labels[:,vocab_size_1:] predictions_2 = predictions[:,vocab_size_1:] # l1 normalization (labels are no less than 0) label_rowsum = tf.maximum( tf.reduce_sum(lables_2, 1, keep_dims=True), epsilon) label_append = 1.0-tf.reduce_max(lables_2, 1, keep_dims=True) norm_float_labels = tf.concat((tf.div(lables_2, label_rowsum),label_append),axis=1) predictions_append = 1.0-tf.reduce_sum(predictions_2, 1, keep_dims=True) softmax_outputs = tf.concat((predictions_2,predictions_append),axis=1) softmax_loss = norm_float_labels * tf.log(softmax_outputs + epsilon) + ( 1 - norm_float_labels) * tf.log(1 - softmax_outputs + epsilon) softmax_loss = tf.negative(tf.reduce_sum(softmax_loss, 1)) return tf.reduce_mean(softmax_loss) + cross_entropy_loss
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="ddcc", original_input=None, dropout=False, keep_prob=None, noise_level=None, num_frames=None, **unused_params): num_supports = FLAGS.num_supports num_models = FLAGS.divergence_model_count support_predictions = [] for i in xrange(num_models): sub_prediction = self.sub_model(model_input,vocab_size, num_mixtures, l2_penalty, sub_scope+"%d"%i, dropout, keep_prob, noise_level) support_predictions.append(sub_prediction) support_predictions = tf.stack(support_predictions, axis=1) main_predictions = tf.reduce_mean(support_predictions, axis=1) return {"predictions": main_predictions, "support_predictions": support_predictions}
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="ddcc", original_input=None, dropout=False, keep_prob=None, noise_level=None, num_frames=None, **unused_params): num_supports = FLAGS.num_supports num_models = FLAGS.divergence_model_count support_predictions = [] for i in xrange(num_models): sub_prediction = self.sub_chain_model(model_input,vocab_size, num_mixtures, l2_penalty, sub_scope+"%d"%i, original_input, dropout, keep_prob, noise_level) support_predictions.append(sub_prediction) support_predictions = tf.stack(support_predictions, axis=1) main_predictions = tf.reduce_mean(support_predictions, axis=1) return {"predictions": main_predictions, "support_predictions": support_predictions}
def calculate_loss(self, predictions, labels, weights=None, **unused_params): with tf.name_scope("loss_xent"): epsilon = 10e-6 if FLAGS.label_smoothing: float_labels = smoothing(labels) else: float_labels = tf.cast(labels, tf.float32) cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + ( 1 - float_labels) * tf.log(1 - predictions + epsilon) cross_entropy_loss = tf.negative(cross_entropy_loss) if weights is not None: print cross_entropy_loss, weights weighted_loss = tf.einsum("ij,i->ij", cross_entropy_loss, weights) print "create weighted_loss", weighted_loss return tf.reduce_mean(tf.reduce_sum(weighted_loss, 1)) else: return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def calculate_loss(self, predictions, support_predictions, labels, **unused_params): """ support_predictions batch_size x num_models x num_classes predictions = tf.reduce_mean(support_predictions, axis=1) """ model_count = tf.shape(support_predictions)[1] vocab_size = tf.shape(support_predictions)[2] mean_predictions = tf.reduce_mean(support_predictions, axis=1, keep_dims=True) support_labels = tf.tile(tf.expand_dims(tf.cast(labels, dtype=tf.float32), axis=1), multiples=[1,model_count,1]) support_means = tf.stop_gradient(tf.tile(mean_predictions, multiples=[1,model_count,1])) support_predictions = tf.reshape(support_predictions, shape=[-1,model_count*vocab_size]) support_labels = tf.reshape(support_labels, shape=[-1,model_count*vocab_size]) support_means = tf.reshape(support_means, shape=[-1,model_count*vocab_size]) ce_loss_fn = CrossEntropyLoss() # The cross entropy between predictions and ground truth cross_entropy_loss = ce_loss_fn.calculate_loss(support_predictions, support_labels, **unused_params) # The cross entropy between predictions and mean predictions divergence = ce_loss_fn.calculate_loss(support_predictions, support_means, **unused_params) loss = cross_entropy_loss * (1.0 - FLAGS.support_loss_percent) - divergence * FLAGS.support_loss_percent return loss
def resolution(self, model_input_raw, num_frames, resolution, method="SELECT"): frame_dim = len(model_input_raw.get_shape()) - 2 feature_dim = len(model_input_raw.get_shape()) - 1 max_frames = model_input_raw.get_shape().as_list()[frame_dim] num_features = model_input_raw.get_shape().as_list()[feature_dim] if resolution > 1: new_max_frames = max_frames / resolution cut_frames = new_max_frames * resolution model_input_raw = model_input_raw[:, :cut_frames, :] model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features]) if method == "MEAN": model_input_raw = tf.reduce_mean(model_input_raw, axis=2) elif method == "MAX": model_input_raw = tf.reduce_max(model_input_raw, axis=2) elif method == "SELECT": model_input_raw = model_input_raw[:,:,resolution-1,:] model_input = tf.nn.l2_normalize(model_input_raw, feature_dim) num_frames = num_frames / resolution else: model_input = tf.nn.l2_normalize(model_input_raw, feature_dim) return model_input, num_frames
def resolution(self, model_input_raw, num_frames, resolution): frame_dim = len(model_input_raw.get_shape()) - 2 feature_dim = len(model_input_raw.get_shape()) - 1 max_frames = model_input_raw.get_shape().as_list()[frame_dim] num_features = model_input_raw.get_shape().as_list()[feature_dim] if resolution > 1: new_max_frames = max_frames / resolution cut_frames = new_max_frames * resolution model_input_raw = model_input_raw[:, :cut_frames, :] model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features]) model_input_raw = tf.reduce_mean(model_input_raw, axis=2) model_input = tf.nn.l2_normalize(model_input_raw, feature_dim) num_frames = num_frames / resolution else: model_input = tf.nn.l2_normalize(model_input_raw, feature_dim) return model_input, num_frames
def resolution(self, model_input_raw, num_frames): resolution = FLAGS.time_resolution frame_dim = len(model_input_raw.get_shape()) - 2 feature_dim = len(model_input_raw.get_shape()) - 1 max_frames = model_input_raw.get_shape().as_list()[frame_dim] num_features = model_input_raw.get_shape().as_list()[feature_dim] new_max_frames = max_frames / resolution cut_frames = new_max_frames * resolution model_input_raw = model_input_raw[:, :cut_frames, :] model_input_raw = tf.reshape(model_input_raw, shape=[-1,new_max_frames,resolution,num_features]) model_input_raw = tf.reduce_mean(model_input_raw, axis=2) num_frames = num_frames / resolution return model_input_raw, num_frames
def FramePooling(frames, method, **unused_params): """Pools over the frames of a video. Args: frames: A tensor with shape [batch_size, num_frames, feature_size]. method: "average", "max", "attention", or "none". Returns: A tensor with shape [batch_size, feature_size] for average, max, or attention pooling. A tensor with shape [batch_size*num_frames, feature_size] for none pooling. Raises: ValueError: if method is other than "average", "max", "attention", or "none". """ if method == "average": return tf.reduce_mean(frames, 1) elif method == "max": return tf.reduce_max(frames, 1) elif method == "none": feature_size = frames.shape_as_list()[2] return tf.reshape(frames, [-1, feature_size]) else: raise ValueError("Unrecognized pooling method: %s" % method)
def model(self, features, labels): x = features["observation"] x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) x = tf.contrib.layers.flatten(x) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) logits = tf.contrib.layers.fully_connected(x, 1, activation_fn=None) prediction = tf.sigmoid(logits) loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.expand_dims(labels, axis=1))) train_op = tf.contrib.layers.optimize_loss( loss, tf.contrib.framework.get_global_step(), optimizer='Adam', learning_rate=0.01) tf.add_to_collection('prediction', prediction) tf.add_to_collection('loss', loss) return prediction, loss, train_op
def build_model2(self): self.weights3, self.biases3 = self.get_en_z_variables() #training Ez self.fake_images = self.generate(self.z, self.y, weights=self.weights1, biases=self.biases1) self.e_z= self.encode_z(self.fake_images, weights=self.weights3, biases=self.biases3) self.loss_z = tf.reduce_mean(tf.square(tf.contrib.layers.flatten(self.e_z - self.z))) t_vars = tf.trainable_variables() self.g_vars = [var for var in t_vars if 'gen' in var.name] self.enz_vars = [var for var in t_vars if 'enz' in var.name] print len(self.g_vars) print len(self.enz_vars) self.saver = tf.train.Saver(self.g_vars) self.saver_z = tf.train.Saver(self.g_vars + self.enz_vars) #Training the Encode_y
def loss(logits, labels): """Add L2Loss to all the trainable variables. Add summary for for "Loss" and "Loss/avg". Args: logits: Logits from inference(). labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size] Returns: Loss tensor of type float. """ # Calculate the average cross entropy loss across the batch. labels = tf.cast(labels, tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return tf.add_n(tf.get_collection('losses'), name='total_loss')
def _add_cross_entropy(labels, logits, pref): """Compute average cross entropy and add to loss collection. Args: labels: Single dimension labels from distorted_inputs() or inputs(). logits: Output map from inference(). pref: Either 'c' or 's', for contours or segments, respectively. """ with tf.variable_scope('{}_cross_entropy'.format(pref)) as scope: class_prop = C_CLASS_PROP if pref == 'c' else S_CLASS_PROP weight_per_label = tf.scalar_mul(class_prop, tf.cast(tf.equal(labels, 0), tf.float32)) + \ tf.scalar_mul(1.0 - class_prop, tf.cast(tf.equal(labels, 1), tf.float32)) cross_entropy = tf.losses.sparse_softmax_cross_entropy( labels=tf.squeeze(labels, squeeze_dims=[3]), logits=logits) cross_entropy_weighted = tf.multiply(weight_per_label, cross_entropy) cross_entropy_mean = tf.reduce_mean(cross_entropy_weighted, name=scope.name) tf.add_to_collection('losses', cross_entropy_mean)
def get_dice_coef(logits, labels): """Compute dice coefficient. Args: logits: Softmax probability applied to fuse layers. labels: Correct annotations (0 or 1). Returns: Mean dice coefficient over full tensor. Source: https://github.com/zsdonghao/tensorlayer/blob/master/tensorlayer/cost.py#L125 """ smooth = 1e-5 inter = tf.reduce_sum(tf.multiply(logits, labels)) l = tf.reduce_sum(logits) r = tf.reduce_sum(labels) return tf.reduce_mean((2.0 * inter + smooth) / (l + r + smooth))
def get_loss(pred, label, end_points, reg_weight=0.001): """ pred: B*NUM_CLASSES, label: B, """ loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred, labels=label) classify_loss = tf.reduce_mean(loss) tf.summary.scalar('classify loss', classify_loss) # Enforce the transformation as orthogonal matrix transform = end_points['transform'] # BxKxK K = transform.get_shape()[1].value mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0,2,1])) mat_diff -= tf.constant(np.eye(K), dtype=tf.float32) mat_diff_loss = tf.nn.l2_loss(mat_diff) tf.summary.scalar('mat loss', mat_diff_loss) return classify_loss + mat_diff_loss * reg_weight
def get_loss(pred, label, end_points, reg_weight=0.001): """ pred: BxNxC, label: BxN, """ loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=pred, labels=label) classify_loss = tf.reduce_mean(loss) tf.scalar_summary('classify loss', classify_loss) # Enforce the transformation as orthogonal matrix transform = end_points['transform'] # BxKxK K = transform.get_shape()[1].value mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0,2,1])) mat_diff -= tf.constant(np.eye(K), dtype=tf.float32) mat_diff_loss = tf.nn.l2_loss(mat_diff) tf.scalar_summary('mat_loss', mat_diff_loss) return classify_loss + mat_diff_loss * reg_weight
def get_loss(l_pred, seg_pred, label, seg, weight, end_points): per_instance_label_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=l_pred, labels=label) label_loss = tf.reduce_mean(per_instance_label_loss) # size of seg_pred is batch_size x point_num x part_cat_num # size of seg is batch_size x point_num per_instance_seg_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=seg_pred, labels=seg), axis=1) seg_loss = tf.reduce_mean(per_instance_seg_loss) per_instance_seg_pred_res = tf.argmax(seg_pred, 2) # Enforce the transformation as orthogonal matrix transform = end_points['transform'] # BxKxK K = transform.get_shape()[1].value mat_diff = tf.matmul(transform, tf.transpose(transform, perm=[0,2,1])) - tf.constant(np.eye(K), dtype=tf.float32) mat_diff_loss = tf.nn.l2_loss(mat_diff) total_loss = weight * seg_loss + (1 - weight) * label_loss + mat_diff_loss * 1e-3 return total_loss, label_loss, per_instance_label_loss, seg_loss, per_instance_seg_loss, per_instance_seg_pred_res
def get_training_tensors(self, learning_rate = 0.001, grad_clip = 5): #----------------------------------------------------------------------- # Build a loss function #----------------------------------------------------------------------- with tf.name_scope('targets-encode'): y_one_hot = tf.one_hot(self.targets, self.n_classes) y_reshaped = tf.reshape(y_one_hot, self.logits.get_shape()) with tf.name_scope('loss'): loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits, labels=y_reshaped) loss = tf.reduce_mean(loss) tf.summary.scalar('loss', loss) #----------------------------------------------------------------------- # Build the optimizer #----------------------------------------------------------------------- with tf.name_scope('optimizer'): tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars), grad_clip) train_op = tf.train.AdamOptimizer(learning_rate) optimizer = train_op.apply_gradients(zip(grads, tvars)) return loss, optimizer
def _smooth_l1_loss(self, bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights, sigma=1.0, dim=[1]): sigma_2 = sigma ** 2 box_diff = bbox_pred - bbox_targets in_box_diff = bbox_inside_weights * box_diff abs_in_box_diff = tf.abs(in_box_diff) smoothL1_sign = tf.stop_gradient(tf.to_float(tf.less(abs_in_box_diff, 1. / sigma_2))) in_loss_box = tf.pow(in_box_diff, 2) * (sigma_2 / 2.) * smoothL1_sign \ + (abs_in_box_diff - (0.5 / sigma_2)) * (1. - smoothL1_sign) out_loss_box = bbox_outside_weights * in_loss_box loss_box = tf.reduce_mean(tf.reduce_sum( out_loss_box, axis=dim )) return loss_box
def batchnormalize(X, eps=1e-8, g=None, b=None): if X.get_shape().ndims == 4: mean = tf.reduce_mean(X, [0,1,2]) std = tf.reduce_mean( tf.square(X-mean), [0,1,2] ) X = (X-mean) / tf.sqrt(std+eps) if g is not None and b is not None: g = tf.reshape(g, [1,1,1,-1]) b = tf.reshape(b, [1,1,1,-1]) X = X*g + b elif X.get_shape().ndims == 2: mean = tf.reduce_mean(X, 0) std = tf.reduce_mean(tf.square(X-mean), 0) X = (X-mean) / tf.sqrt(std+eps) if g is not None and b is not None: g = tf.reshape(g, [1,-1]) b = tf.reshape(b, [1,-1]) X = X*g + b else: raise NotImplementedError return X
def build_model(self): Z = tf.placeholder(tf.float32, [self.batch_size, self.dim_z]) Y = tf.placeholder(tf.float32, [self.batch_size, self.dim_y]) image_real = tf.placeholder(tf.float32, [self.batch_size]+self.image_shape) h4 = self.generate(Z,Y) #image_gen comes from sigmoid output of generator image_gen = tf.nn.sigmoid(h4) raw_real2 = self.discriminate(image_real, Y) #p_real = tf.nn.sigmoid(raw_real) p_real=tf.reduce_mean(raw_real2) raw_gen2 = self.discriminate(image_gen, Y) #p_gen = tf.nn.sigmoid(raw_gen) p_gen = tf.reduce_mean(raw_gen2) discrim_cost = tf.reduce_sum(raw_real2) - tf.reduce_sum(raw_gen2) gen_cost = -tf.reduce_mean(raw_gen2) return Z, Y, image_real, discrim_cost, gen_cost, p_real, p_gen
def __init__(self, channels=3, n_class=2, cost="cross_entropy", cost_kwargs={}, **kwargs): tf.reset_default_graph() self.n_class = n_class self.summaries = kwargs.get("summaries", True) self.x = tf.placeholder("float", shape=[None, None, None, channels]) self.y = tf.placeholder("float", shape=[None, None, None, n_class]) self.keep_prob = tf.placeholder(tf.float32) #dropout (keep probability) logits, self.variables, self.offset = create_conv_net(self.x, self.keep_prob, channels, n_class, **kwargs) self.cost = self._get_cost(logits, cost, cost_kwargs) self.gradients_node = tf.gradients(self.cost, self.variables) self.cross_entropy = tf.reduce_mean(cross_entropy(tf.reshape(self.y, [-1, n_class]), tf.reshape(pixel_wise_softmax_2(logits), [-1, n_class]))) self.predicter = pixel_wise_softmax_2(logits) self.correct_pred = tf.equal(tf.argmax(self.predicter, 3), tf.argmax(self.y, 3)) self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))
def _build_graph(self, image_size): self.image_size = image_size self.images = tf.placeholder(tf.float32, shape = (None, image_size, image_size, 3)) images_mini = tf.image.resize_images(self.images, size = (int(image_size/4), int(image_size/4))) self.images_blur = tf.image.resize_images(images_mini, size = (image_size, image_size)) self.net = U_Net(output_ch = 3, block_fn = 'origin') self.images_reconst = self.net(self.images_blur, reuse = False) # self.image_reconst can be [-inf +inf], so need to clip its value if visualize them as images. self.loss = tf.reduce_mean((self.images_reconst - self.images)**2) self.opt = tf.train.AdamOptimizer()\ .minimize(self.loss, var_list = self.net.vars) self.saver = tf.train.Saver() self.sess.run(tf.global_variables_initializer())
def build_model(self): Gen=GeneratorTypes[self.gan_type] config=self.config self.gen=Gen(config.batch_size,config.gen_hidden_size,config.gen_z_dim) with tf.variable_scope('Disc') as scope: self.D1 = Discriminator(self.data.X, config.disc_hidden_size) scope.reuse_variables() self.D2 = Discriminator(self.gen.X, config.disc_hidden_size) d_var = tf.contrib.framework.get_variables(scope) d_loss_real=tf.reduce_mean( sxe(self.D1,1) ) d_loss_fake=tf.reduce_mean( sxe(self.D2,0) ) self.loss_d = d_loss_real + d_loss_fake self.loss_g = tf.reduce_mean( sxe(self.D2,1) ) optimizer=tf.train.AdamOptimizer g_optimizer=optimizer(self.config.lr_gen) d_optimizer=optimizer(self.config.lr_disc) self.opt_d = d_optimizer.minimize(self.loss_d,var_list= d_var) self.opt_g = g_optimizer.minimize(self.loss_g,var_list= self.gen.tr_var, global_step=self.gen.step) with tf.control_dependencies([self.inc_step]): self.train_op=tf.group(self.opt_d,self.opt_g)
def Grad_Penalty(real_data,fake_data,Discriminator,config): ''' Implemention from "Improved training of Wasserstein" Interpolation based estimation of the gradient of the discriminator. Used to penalize the derivative rather than explicitly constrain lipschitz. ''' batch_size=config.batch_size LAMBDA=config.lambda_W n_hidden=config.critic_hidden_size alpha = tf.random_uniform([batch_size,1],0.,1.) interpolates = alpha*real_data + ((1-alpha)*fake_data)#Could do more if not fixed batch_size disc_interpolates = Discriminator(interpolates,batch_size,n_hidden=n_hidden,config=config, reuse=True)[1]#logits gradients = tf.gradients(disc_interpolates,[interpolates])[0]#orig slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1])) gradient_penalty = tf.reduce_mean((slopes-1)**2) grad_cost = LAMBDA*gradient_penalty return grad_cost,slopes
def _get_loss(self,labels): with tf.name_scope("Loss"): """ with tf.name_scope("logloss"): logit = tf.squeeze(tf.nn.sigmoid(self.logit)) self.loss = tf.reduce_mean(self._logloss(labels, logit)) """ with tf.name_scope("L2_loss"): if self.flags.lambdax: lambdax = self.flags.lambdax else: lambdax = 0 self.l2loss = lambdax*tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) with tf.name_scope("dice_coef"): #yp_label = tf.cast(logit>self.flags.threshold, tf.float32) logit = tf.squeeze(self.logit) self.acc = tf.reduce_mean(self._dice_coef(labels,logit)) self.metric = "dice_coef" self.loss = -self.acc with tf.name_scope("summary"): if self.flags.visualize: tf.summary.scalar(name='dice coef', tensor=self.acc, collections=[tf.GraphKeys.SCALARS])
def sparse_cross_entropy_loss(logits, labels, weight=1.0, scope=None): """Define a Cross Entropy loss using sparse_softmax_cross_entropy_with_logits. It can scale the loss by weight factor, and smooth the labels. Args: logits: [batch_size, num_classes] logits outputs of the network . labels: [batch_size,] target labels. weight: scale the loss by this factor. scope: Optional scope for op_scope. Returns: A tensor with the softmax_cross_entropy loss. """ with tf.op_scope([logits, labels], scope, 'SparseCrossEntropyLoss'): cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits,labels,name='xentropy') weight = tf.convert_to_tensor(weight, dtype=logits.dtype.base_dtype, name='loss_weight') loss = tf.mul(weight, tf.reduce_mean(cross_entropy), name='value') tf.add_to_collection(LOSSES_COLLECTION, loss) return loss
def get_label_costs(coder, dataset, labels, batch_size=100): """ Return average cross entropy loss and class error rate on dataset by coder object with its current weights. """ n_batches = dataset.shape[0] // batch_size error = 0. cost = 0. for index in range(n_batches): batch = dataset[index * batch_size : (index+1) * batch_size] labels_batch = labels[index * batch_size : (index+1) * batch_size] predicted = coder.get_hidden_values(batch) loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=predicted, labels=labels_batch) cost += tf.reduce_mean(loss).eval() bad_prediction = tf.not_equal(tf.argmax(predicted , 1), labels_batch) error += tf.reduce_mean(tf.cast(bad_prediction, tf.float32)).eval() return (cost / n_batches, error / n_batches)
def recode_cost(self, inputs, variation, eps=1e-5, **kwargs): """ Cost for given input batch of samples, under current params. """ h = self.get_h_inputs(inputs) z_mu = tf.matmul(h, self.params['Mhz']) + self.params['bMhz'] z_sig = tf.matmul(h, self.params['Shz']) + self.params['bShz'] # KL divergence between latent space induced by encoder and ... lat_loss = -tf.reduce_sum(1 + z_sig - z_mu**2 - tf.exp(z_sig), 1) z = z_mu + tf.sqrt(tf.exp(z_sig)) * variation h = self.get_h_latents(z) x_mu = self.decoding(tf.matmul(h, self.params['Mhx']) + self.params['bMhx']) x_sig = self.decoding(tf.matmul(h, self.params['Shx']) + self.params['bShx']) # x_sig = tf.clip_by_value(x_mu * (1 - x_mu), .05, 1) # decoding likelihood term like_loss = tf.reduce_sum(tf.log(x_sig + eps) + (inputs - x_mu)**2 / x_sig, 1) # # Mean cross entropy between input and encode-decoded input. # like_loss = 2 * tf.reduce_sum(functions.cross_entropy(inputs, x_mu), 1) return .5 * tf.reduce_mean(like_loss + lat_loss)
def add_evaluation_step(graph, final_tensor_name, ground_truth_tensor_name): """Inserts the operations we need to evaluate the accuracy of our results. Args: graph: Container for the existing model's Graph. final_tensor_name: Name string for the new final node that produces results. ground_truth_tensor_name: Name string for the node we feed ground truth data into. Returns: Nothing. """ result_tensor = graph.get_tensor_by_name(ensure_name_has_port( final_tensor_name)) ground_truth_tensor = graph.get_tensor_by_name(ensure_name_has_port( ground_truth_tensor_name)) correct_prediction = tf.equal( tf.argmax(result_tensor, 1), tf.argmax(ground_truth_tensor, 1)) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, 'float')) return evaluation_step
def loss(self, l2_lambda=0.0001): # 0.001 with tf.name_scope("loss"): # input: `logits`:[batch_size, num_classes], and `labels`:[batch_size] # output: A 1-D `Tensor` of length `batch_size` of the same type as `logits` with the softmax cross entropy loss. losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.input_y_label,logits=self.logits); # sigmoid_cross_entropy_with_logits.#losses=tf.nn.softmax_cross_entropy_with_logits(labels=self.input_y,logits=self.logits) # print("1.sparse_softmax_cross_entropy_with_logits.losses:",losses) # shape=(?,) loss = tf.reduce_mean(losses) # print("2.loss.loss:", loss) #shape=() l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if ('bias' not in v.name ) and ('alpha' not in v.name)]) * l2_lambda loss = loss + l2_losses return loss #def loss_seq2seq(self): # with tf.variable_scope("loss"): # losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.input_y_label, logits=self.logits);#losses:[batch_size,self.decoder_sent_length] # loss_batch=tf.reduce_sum(losses,axis=1)/self.decoder_sent_length #loss_batch:[batch_size] # loss=tf.reduce_mean(loss_batch) # l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name]) * self.l2_lambda # loss = loss + l2_losses # return loss
def layer_normalization(self,x): """ x should be:[batch_size,sequence_length,d_model] :return: """ filter=x.get_shape()[-1] #last dimension of x. e.g. 512 print("layer_normalization:==================>variable_scope:","layer_normalization"+str(self.layer_index)+self.type) with tf.variable_scope("layer_normalization"+str(self.layer_index)+self.type): # 1. normalize input by using mean and variance according to last dimension mean=tf.reduce_mean(x,axis=-1,keep_dims=True) #[batch_size,sequence_length,1] variance=tf.reduce_mean(tf.square(x-mean),axis=-1,keep_dims=True) #[batch_size,sequence_length,1] norm_x=(x-mean)*tf.rsqrt(variance+1e-6) #[batch_size,sequence_length,d_model] # 2. re-scale normalized input back scale=tf.get_variable("layer_norm_scale",[filter],initializer=tf.ones_initializer) #[filter] bias=tf.get_variable("layer_norm_bias",[filter],initializer=tf.ones_initializer) #[filter] output=norm_x*scale+bias #[batch_size,sequence_length,d_model] return output #[batch_size,sequence_length,d_model]
def critic_loss(self): return tf.reduce_mean(tf.square(self.TD_loss))
def actor_loss(self): if self.config.mode == 'discrete': log_prob = tf.reduce_sum(tf.log(self.a_prob) * tf.one_hot(self.action_input, self.action_dim, dtype=tf.float32), axis=1, keep_dims=True) # use entropy to encourage exploration exp_v = log_prob * self.TD_loss entropy = -tf.reduce_sum(self.a_prob * tf.log(self.a_prob), axis=1, keep_dims=True) # encourage exploration exp_v = self.config.ENTROPY_BETA * entropy + exp_v return tf.reduce_mean(-exp_v) # ????????log_prb????????????????????TD_loss elif self.config.mode == 'continuous': log_prob = self.action_normal_dist.log_prob(self.action_input) exp_v = log_prob * self.TD_loss # use entropy to encourage exploration exp_v = self.config.ENTROPY_BETA * self.action_normal_dist.entropy() + exp_v return tf.reduce_mean(-exp_v)
def make_skipgram_softmax_loss(embeddings_matrix, vocabulary_size, vector_size): vectors = tf.get_variable('vectors', (vocabulary_size, vector_size), dtype=tf.float32, initializer=tf.constant_initializer(embeddings_matrix)) minibatch = tf.placeholder(shape=(None, 2), dtype=tf.int32) center_word_vector = tf.nn.embedding_lookup(vectors, minibatch[:,0]) yhat = tf.matmul(center_word_vector, vectors, transpose_b=True) predict_word = minibatch[:,1] loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=predict_word, logits=yhat) loss = tf.reduce_mean(loss) return vectors, minibatch, loss
def pca_fit(X, n_components): mean = tf.reduce_mean(X, axis=0) centered_X = X - mean S, U, V = tf.svd(centered_X) return V[:n_components], mean
def decov_loss(xs): """Decov loss as described in https://arxiv.org/pdf/1511.06068.pdf 'Reducing Overfitting In Deep Networks by Decorrelating Representation' """ x = tf.reshape(xs, [int(xs.get_shape()[0]), -1]) m = tf.reduce_mean(x, 0, True) z = tf.expand_dims(x-m, 2) corr = tf.reduce_mean(tf.matmul(z, tf.transpose(z, perm=[0,2,1])), 0) corr_frob_sqr = tf.reduce_sum(tf.square(corr)) corr_diag_sqr = tf.reduce_sum(tf.square(tf.diag_part(corr))) loss = 0.5*(corr_frob_sqr - corr_diag_sqr) return loss
