我们从Python开源项目中，提取了以下36个代码示例，用于说明如何使用keras.backend.l2_normalize()。
def call(self, inputs): x1 = inputs[0] x2 = inputs[1] if self.match_type in ['dot']: if self.normalize: x1 = K.l2_normalize(x1, axis=2) x2 = K.l2_normalize(x2, axis=2) output = K.tf.einsum('abd,acd->abc', x1, x2) output = K.tf.expand_dims(output, 3) elif self.match_type in ['mul', 'plus', 'minus']: x1_exp = K.tf.stack([x1] * self.shape2[1], 2) x2_exp = K.tf.stack([x2] * self.shape1[1], 1) if self.match_type == 'mul': output = x1_exp * x2_exp elif self.match_type == 'plus': output = x1_exp + x2_exp elif self.match_type == 'minus': output = x1_exp - x2_exp elif self.match_type in ['concat']: x1_exp = K.tf.stack([x1] * self.shape2[1], axis=2) x2_exp = K.tf.stack([x2] * self.shape1[1], axis=1) output = K.tf.concat([x1_exp, x2_exp], axis=3) return output
def define_network(vector_size, loss): base_model = InceptionV3(weights='imagenet', include_top=True) for layer in base_model.layers: # Freeze layers in pretrained model layer.trainable = False # fully-connected layer to predict x = Dense(4096, activation='relu', name='fc1')(base_model.layers[-2].output) x = Dense(8096, activation='relu', name='fc2')(x) x = Dropout(0.5)(x) x = Dense(2048,activation='relu', name='fc3')(x) predictions = Dense(vector_size, activation='relu')(x) l2 = Lambda(lambda x: K.l2_normalize(x, axis=1))(predictions) model = Model(inputs=base_model.inputs, outputs=l2) optimizer = 'adam' if loss == 'euclidean': model.compile(optimizer = optimizer, loss = euclidean_distance) else: model.compile(optimizer = optimizer, loss = loss) return model
def create_full_matching_layer_f(self, input_dim_a, input_dim_b): """Create a full-matching layer of a model.""" inp_a = Input(shape=(input_dim_a, self.hidden_dim,)) inp_b = Input(shape=(input_dim_b, self.hidden_dim,)) W = [] for i in range(self.perspective_num): wi = K.random_uniform_variable((1, self.hidden_dim), -1.0, 1.0, seed=self.seed if self.seed is not None else 243) W.append(wi) val = np.concatenate((np.zeros((self.max_sequence_length-1,1)), np.ones((1,1))), axis=0) kcon = K.constant(value=val, dtype='float32') inp_b_perm = Lambda(lambda x: K.permute_dimensions(x, (0,2,1)))(inp_b) last_state = Lambda(lambda x: K.permute_dimensions(K.dot(x, kcon), (0,2,1)))(inp_b_perm) m = [] for i in range(self.perspective_num): outp_a = Lambda(lambda x: x * W[i])(inp_a) outp_last = Lambda(lambda x: x * W[i])(last_state) outp_a = Lambda(lambda x: K.l2_normalize(x, -1))(outp_a) outp_last = Lambda(lambda x: K.l2_normalize(x, -1))(outp_last) outp_last = Lambda(lambda x: K.permute_dimensions(x, (0,2,1)))(outp_last) outp = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([outp_last, outp_a]) outp = Lambda(lambda x: K.permute_dimensions(x, (0,2,1)))(outp) m.append(outp) if self.perspective_num > 1: persp = Lambda(lambda x: K.concatenate(x, 2))(m) else: persp = m model = Model(inputs=[inp_a, inp_b], outputs=persp) return model
def create_full_matching_layer_b(self, input_dim_a, input_dim_b): """Create a full-matching layer of a model.""" inp_a = Input(shape=(input_dim_a, self.hidden_dim,)) inp_b = Input(shape=(input_dim_b, self.hidden_dim,)) W = [] for i in range(self.perspective_num): wi = K.random_uniform_variable((1, self.hidden_dim), -1.0, 1.0, seed=self.seed if self.seed is not None else 243) W.append(wi) val = np.concatenate((np.ones((1, 1)), np.zeros((self.max_sequence_length - 1, 1))), axis=0) kcon = K.constant(value=val, dtype='float32') inp_b_perm = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(inp_b) last_state = Lambda(lambda x: K.permute_dimensions(K.dot(x, kcon), (0, 2, 1)))(inp_b_perm) m = [] for i in range(self.perspective_num): outp_a = Lambda(lambda x: x * W[i])(inp_a) outp_last = Lambda(lambda x: x * W[i])(last_state) outp_a = Lambda(lambda x: K.l2_normalize(x, -1))(outp_a) outp_last = Lambda(lambda x: K.l2_normalize(x, -1))(outp_last) outp_last = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(outp_last) outp = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([outp_last, outp_a]) outp = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(outp) m.append(outp) if self.perspective_num > 1: persp = Lambda(lambda x: K.concatenate(x, 2))(m) else: persp = m model = Model(inputs=[inp_a, inp_b], outputs=persp) return model
def create_maxpool_matching_layer(self, input_dim_a, input_dim_b): """Create a maxpooling-matching layer of a model.""" inp_a = Input(shape=(input_dim_a, self.hidden_dim,)) inp_b = Input(shape=(input_dim_b, self.hidden_dim,)) W = [] for i in range(self.perspective_num): wi = K.random_uniform_variable((1, self.hidden_dim), -1.0, 1.0, seed=self.seed if self.seed is not None else 243) W.append(wi) m = [] for i in range(self.perspective_num): outp_a = Lambda(lambda x: x * W[i])(inp_a) outp_b = Lambda(lambda x: x * W[i])(inp_b) outp_a = Lambda(lambda x: K.l2_normalize(x, -1))(outp_a) outp_b = Lambda(lambda x: K.l2_normalize(x, -1))(outp_b) outp_b = Lambda(lambda x: K.permute_dimensions(x, (0,2,1)))(outp_b) outp = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([outp_b, outp_a]) outp = Lambda(lambda x: K.permute_dimensions(x, (0,2,1)))(outp) outp = Lambda(lambda x: K.max(x, -1, keepdims=True))(outp) m.append(outp) if self.perspective_num > 1: persp = Lambda(lambda x: K.concatenate(x, 2))(m) else: persp = m model = Model(inputs=[inp_a, inp_b], outputs=persp) return model
def create_maxatt_matching_layer(self, input_dim_a, input_dim_b): """Create a max-attentive-matching layer of a model.""" inp_a = Input(shape=(input_dim_a, self.hidden_dim,)) inp_b = Input(shape=(input_dim_b, self.hidden_dim,)) W = [] for i in range(self.perspective_num): wi = K.random_uniform_variable((1, self.hidden_dim), -1.0, 1.0, seed=self.seed if self.seed is not None else 243) W.append(wi) outp_a = Lambda(lambda x: K.l2_normalize(x, -1))(inp_a) outp_b = Lambda(lambda x: K.l2_normalize(x, -1))(inp_b) outp_b = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(outp_b) alpha = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([outp_b, outp_a]) alpha = Lambda(lambda x: K.one_hot(K.argmax(x, 1), self.max_sequence_length))(alpha) hmax = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([alpha, outp_b]) m = [] for i in range(self.perspective_num): outp_a = Lambda(lambda x: x * W[i])(inp_a) outp_hmax = Lambda(lambda x: x * W[i])(hmax) outp_a = Lambda(lambda x: K.l2_normalize(x, -1))(outp_a) outp_hmax = Lambda(lambda x: K.l2_normalize(x, -1))(outp_hmax) outp_hmax = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(outp_hmax) outp = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([outp_hmax, outp_a]) val = np.eye(self.max_sequence_length) kcon = K.constant(value=val, dtype='float32') outp = Lambda(lambda x: K.sum(x * kcon, -1, keepdims=True))(outp) m.append(outp) if self.perspective_num > 1: persp = Lambda(lambda x: K.concatenate(x, 2))(m) else: persp = m model = Model(inputs=[inp_a, inp_b], outputs=persp) return model
def call(self, inputs): x1 = inputs[0] x2 = inputs[1] if self.normalize: x1 = K.l2_normalize(x1, axis=2) x2 = K.l2_normalize(x2, axis=2) output = K.tf.einsum('abd,fde,ace->afbc', x1, self.M, x2) return output
def call(self, x, mask=None): x -= K.mean(x, axis=1, keepdims=True) x = K.l2_normalize(x, axis=1) pos = K.relu(x) neg = K.relu(-x) return K.concatenate([pos, neg], axis=1) # global parameters
def call(self, inputs): inputs -= K.mean(inputs, axis=1, keepdims=True) inputs = K.l2_normalize(inputs, axis=1) pos = K.relu(inputs) neg = K.relu(-inputs) return K.concatenate([pos, neg], axis=1) # global parameters
def cosine(a, b): """Cosine similarity. Maximum is 1 (a == b), minimum is -1 (a == -b).""" a = K.l2_normalize(a) b = K.l2_normalize(b) return 1 - K.mean(a * b, axis=-1)
def _initialize_centers(self, n_labels, output_dim): trigger = Input(shape=(n_labels, ), name="trigger") x = Dense(output_dim, activation='linear', name='dense')(trigger) centers = Lambda(lambda x: K.l2_normalize(x, axis=-1), output_shape=(output_dim, ), name="centers")(x) model = Model(inputs=trigger, outputs=centers) model.compile(optimizer=SSMORMS3(), loss=self.loss) return model
def call(self, x, mask=None): # thanks to L2 normalization, mask actually has no effect return K.l2_normalize(K.sum(x, axis=1), axis=-1)
def on_train_begin(self, logs=None): # number of classes n_classes = logs['n_classes'] if logs['restart']: weights_h5 = self.WEIGHTS_H5.format(log_dir=logs['log_dir'], epoch=logs['epoch']) self.centers_ = keras.models.load_model( weights_h5, custom_objects=CUSTOM_OBJECTS, compile=True) else: # dimension of embedding space output_dim = self.model.output_shape[-1] # centers model trigger = Input(shape=(n_classes, ), name="trigger") x = Dense(output_dim, activation='linear', name='dense')(trigger) centers = Lambda(lambda x: K.l2_normalize(x, axis=-1), output_shape=(output_dim, ), name="centers")(x) self.centers_ = Model(inputs=trigger, outputs=centers) self.centers_.compile(optimizer=SSMORMS3(), loss=precomputed_gradient_loss) # save list of classes centers_txt = self.CENTERS_TXT.format(**logs) with open(centers_txt, mode='w') as fp: for label in logs['classes']: fp.write('{label}\n'.format(label=label.decode('utf8'))) self.trigger_ = np.eye(n_classes) self.fC_ = self.centers_.predict(self.trigger_).astype(self.float_autograd_)
def call(self, x, mask=None): output = K.l2_normalize(x, self.axis) output *= self.gamma return output
def l2_normalize(x): return K.l2_normalize(x, 0)
def squared_root_normalization(x): """ Squared root normalization for convolution layers` output first apply global average pooling followed by squared root on all elements then l2 normalize the vector :param x: input tensor, output of convolution layer :return: """ x = GlobalAveragePooling2D()(x) #output shape = (None, nc) # x = K.sqrt(x) #x = K.l2_normalize(x, axis=0) return x
def get_model_3(params): # metadata inputs2 = Input(shape=(params["n_metafeatures"],)) x2 = Dropout(params["dropout_factor"])(inputs2) if params["n_dense"] > 0: dense2 = Dense(output_dim=params["n_dense"], init="uniform", activation='relu') x2 = dense2(x2) logging.debug("Output CNN: %s" % str(dense2.output_shape)) x2 = Dropout(params["dropout_factor"])(x2) if params["n_dense_2"] > 0: dense3 = Dense(output_dim=params["n_dense_2"], init="uniform", activation='relu') x2 = dense3(x2) logging.debug("Output CNN: %s" % str(dense3.output_shape)) x2 = Dropout(params["dropout_factor"])(x2) dense4 = Dense(output_dim=params["n_out"], init="uniform", activation=params['final_activation']) xout = dense4(x2) logging.debug("Output CNN: %s" % str(dense4.output_shape)) if params['final_activation'] == 'linear': reg = Lambda(lambda x :K.l2_normalize(x, axis=1)) xout = reg(xout) model = Model(input=inputs2, output=xout) return model # Metadata 2 inputs, post-merge with dense layers
def get_model_32(params): # metadata inputs = Input(shape=(params["n_metafeatures"],)) reg = Lambda(lambda x :K.l2_normalize(x, axis=1)) x1 = reg(inputs) inputs2 = Input(shape=(params["n_metafeatures2"],)) reg2 = Lambda(lambda x :K.l2_normalize(x, axis=1)) x2 = reg2(inputs2) # merge x = merge([x1, x2], mode='concat', concat_axis=1) x = Dropout(params["dropout_factor"])(x) if params['n_dense'] > 0: dense2 = Dense(output_dim=params["n_dense"], init="uniform", activation='relu') x = dense2(x) logging.debug("Output CNN: %s" % str(dense2.output_shape)) dense4 = Dense(output_dim=params["n_out"], init="uniform", activation=params['final_activation']) xout = dense4(x) logging.debug("Output CNN: %s" % str(dense4.output_shape)) if params['final_activation'] == 'linear': reg = Lambda(lambda x :K.l2_normalize(x, axis=1)) xout = reg(xout) model = Model(input=[inputs,inputs2], output=xout) return model # Metadata 3 inputs, pre-merge and l2
def get_model_33(params): # metadata inputs = Input(shape=(params["n_metafeatures"],)) reg = Lambda(lambda x :K.l2_normalize(x, axis=1)) x1 = reg(inputs) inputs2 = Input(shape=(params["n_metafeatures2"],)) reg2 = Lambda(lambda x :K.l2_normalize(x, axis=1)) x2 = reg2(inputs2) inputs3 = Input(shape=(params["n_metafeatures3"],)) reg3 = Lambda(lambda x :K.l2_normalize(x, axis=1)) x3 = reg3(inputs3) # merge x = merge([x1, x2, x3], mode='concat', concat_axis=1) x = Dropout(params["dropout_factor"])(x) if params['n_dense'] > 0: dense2 = Dense(output_dim=params["n_dense"], init="uniform", activation='relu') x = dense2(x) logging.debug("Output CNN: %s" % str(dense2.output_shape)) dense4 = Dense(output_dim=params["n_out"], init="uniform", activation=params['final_activation']) xout = dense4(x) logging.debug("Output CNN: %s" % str(dense4.output_shape)) if params['final_activation'] == 'linear': reg = Lambda(lambda x :K.l2_normalize(x, axis=1)) xout = reg(xout) model = Model(input=[inputs,inputs2,inputs3], output=xout) return model
def call(self, x, mask=None): x -= K.mean(x, axis=1, keepdims=True) x = K.l2_normalize(x, axis=1) pos = K.relu(x) neg = K.relu(-x) return K.concatenate([pos, neg], axis=1) #Result dictionary
def _cosine_distance(M, k): # this is equation (6), or as I like to call it: The NaN factory. # TODO: Find it in a library (keras cosine loss?) # normalizing first as it is better conditioned. nk = K.l2_normalize(k, axis=-1) nM = K.l2_normalize(M, axis=-1) cosine_distance = K.batch_dot(nM, nk) # TODO: Do succesfull error handling #cosine_distance_error_handling = tf.Print(cosine_distance, [cosine_distance], message="NaN occured in _cosine_distance") #cosine_distance_error_handling = K.ones(cosine_distance_error_handling.shape) #cosine_distance = tf.case({K.any(tf.is_nan(cosine_distance)) : (lambda: cosine_distance_error_handling)}, # default = lambda: cosine_distance, strict=True) return cosine_distance
def cosine_similarity(x, y): x = K.l2_normalize(x, axis=-1) y = K.l2_normalize(y, axis=-1) return K.sum(x * y, axis=-1)
def get_metric(): def return_metric(): import keras.backend as K def cosine_proximity(y_true, y_pred): y_true = K.l2_normalize(y_true, axis=-1) y_pred = K.l2_normalize(y_pred, axis=-1) return -K.mean(y_true * y_pred) return cosine_proximity return return_metric
def build_tpe(n_in, n_out, W_pca=None): a = Input(shape=(n_in,)) p = Input(shape=(n_in,)) n = Input(shape=(n_in,)) if W_pca is None: W_pca = np.zeros((n_in, n_out)) base_model = Sequential() base_model.add(Dense(n_out, input_dim=n_in, bias=False, weights=[W_pca], activation='linear')) base_model.add(Lambda(lambda x: K.l2_normalize(x, axis=1))) # base_model = Sequential() # base_model.add(Dense(178, input_dim=n_in, bias=True, activation='relu')) # base_model.add(Dense(n_out, bias=True, activation='tanh')) # base_model.add(Lambda(lambda x: K.l2_normalize(x, axis=1))) a_emb = base_model(a) p_emb = base_model(p) n_emb = base_model(n) e = merge([a_emb, p_emb, n_emb], mode=triplet_merge, output_shape=triplet_merge_shape) model = Model(input=[a, p, n], output=e) predict = Model(input=a, output=a_emb) model.compile(loss=triplet_loss, optimizer='rmsprop') return model, predict
def create_att_matching_layer(self, input_dim_a, input_dim_b): """Create an attentive-matching layer of a model.""" inp_a = Input(shape=(input_dim_a, self.hidden_dim,)) inp_b = Input(shape=(input_dim_b, self.hidden_dim,)) w = [] for i in range(self.perspective_num): wi = K.random_uniform_variable((1, self.hidden_dim), -1.0, 1.0, seed=self.seed if self.seed is not None else 243) w.append(wi) outp_a = Lambda(lambda x: K.l2_normalize(x, -1))(inp_a) outp_b = Lambda(lambda x: K.l2_normalize(x, -1))(inp_b) outp_b = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(outp_b) alpha = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([outp_b, outp_a]) alpha = Lambda(lambda x: K.l2_normalize(x, 1))(alpha) hmean = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([alpha, outp_b]) m = [] for i in range(self.perspective_num): outp_a = Lambda(lambda x: x * w[i])(inp_a) outp_hmean = Lambda(lambda x: x * w[i])(hmean) outp_a = Lambda(lambda x: K.l2_normalize(x, -1))(outp_a) outp_hmean = Lambda(lambda x: K.l2_normalize(x, -1))(outp_hmean) outp_hmean = Lambda(lambda x: K.permute_dimensions(x, (0, 2, 1)))(outp_hmean) outp = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=[1, 2]))([outp_hmean, outp_a]) val = np.eye(self.max_sequence_length) kcon = K.constant(value=val, dtype='float32') outp = Lambda(lambda x: K.sum(x * kcon, -1, keepdims=True))(outp) m.append(outp) if self.perspective_num > 1: persp = Lambda(lambda x: K.concatenate(x, 2))(m) else: persp = m model = Model(inputs=[inp_a, inp_b], outputs=persp) return model
def model2(weights_path=None): ''' Creates a model by concatenating the features from lower layers with high level convolution features for all aesthetic attributes along with overall aesthetic score :param weights_path: path of the weight file :return: Keras model instance This is the model used in the paper ''' _input = Input(shape=(299, 299, 3)) resnet = ResNet50(include_top=False, weights='imagenet', input_tensor=_input) activation_layers = [] layers = resnet.layers for layer in layers: # print layer.name, layer.input_shape, layer.output_shape if 'activation' in layer.name: activation_layers.append(layer) activations = 0 activation_plus_squared_outputs = [] # Remove last activation layer so # it can be used with spatial pooling layer if required nlayers = len(activation_layers) - 1 for i in range(1, nlayers): layer = activation_layers[i] if layer.output_shape[-1] > activation_layers[i - 1].output_shape[-1]: # print layer.name, layer.input_shape, layer.output_shape activations += layer.output_shape[-1] _out = Lambda(squared_root_normalization, output_shape=squared_root_normalization_output_shape, name=layer.name + '_normalized')(layer.output) activation_plus_squared_outputs.append(_out) # print "sum of all activations should be {}".format(activations) last_layer_output = GlobalAveragePooling2D()(activation_layers[-1].output) # last_layer_output = Lambda(K.sqrt, output_shape=squared_root_normalization_output_shape)(last_layer_output) last_layer_output = Lambda(l2_normalize, output_shape=l2_normalize_output_shape, name=activation_layers[-1].name+'_normalized')(last_layer_output) activation_plus_squared_outputs.append(last_layer_output) merged = merge(activation_plus_squared_outputs, mode='concat', concat_axis=1) merged = Lambda(l2_normalize, output_shape=l2_normalize_output_shape, name='merge')(merged) # output of model outputs = [] attrs = ['BalacingElements', 'ColorHarmony', 'Content', 'DoF', 'Light', 'MotionBlur', 'Object', 'RuleOfThirds', 'VividColor'] for attribute in attrs: outputs.append(Dense(1, init='glorot_uniform', activation='tanh', name=attribute)(merged)) non_negative_attrs = ['Repetition', 'Symmetry', 'score'] for attribute in non_negative_attrs: outputs.append(Dense(1, init='glorot_uniform', activation='sigmoid', name=attribute)(merged)) model = Model(input=_input, output=outputs) if weights_path: model.load_weights(weights_path) return model
def get_model_31(params): # metadata inputs = Input(shape=(params["n_metafeatures"],)) norm = BatchNormalization() x = norm(inputs) x = Dropout(params["dropout_factor"])(x) dense = Dense(output_dim=params["n_dense"], init="uniform", activation='relu') x = dense(x) logging.debug("Output CNN: %s" % str(dense.output_shape)) x = Dropout(params["dropout_factor"])(x) inputs2 = Input(shape=(params["n_metafeatures2"],)) norm2 = BatchNormalization() x2 = norm2(inputs2) x2 = Dropout(params["dropout_factor"])(x2) dense2 = Dense(output_dim=params["n_dense"], init="uniform", activation='relu') x2 = dense2(x2) logging.debug("Output CNN: %s" % str(dense2.output_shape)) x2 = Dropout(params["dropout_factor"])(x2) # merge xout = merge([x, x2], mode='concat', concat_axis=1) dense4 = Dense(output_dim=params["n_out"], init="uniform", activation=params['final_activation']) xout = dense4(xout) logging.debug("Output CNN: %s" % str(dense4.output_shape)) if params['final_activation'] == 'linear': reg = Lambda(lambda x :K.l2_normalize(x, axis=1)) xout = reg(xout) model = Model(input=[inputs,inputs2], output=xout) return model # Metadata 2 inputs, pre-merge and l2
def get_model_4(params): embedding_weights = pickle.load(open(common.TRAINDATA_DIR+"/embedding_weights_w2v_%s.pk" % params['embeddings_suffix'],"rb")) graph_in = Input(shape=(params['sequence_length'], params['embedding_dim'])) convs = [] for fsz in params['filter_sizes']: conv = Convolution1D(nb_filter=params['num_filters'], filter_length=fsz, border_mode='valid', activation='relu', subsample_length=1) x = conv(graph_in) logging.debug("Filter size: %s" % fsz) logging.debug("Output CNN: %s" % str(conv.output_shape)) pool = GlobalMaxPooling1D() x = pool(x) logging.debug("Output Pooling: %s" % str(pool.output_shape)) convs.append(x) if len(params['filter_sizes'])>1: merge = Merge(mode='concat') out = merge(convs) logging.debug("Merge: %s" % str(merge.output_shape)) else: out = convs[0] graph = Model(input=graph_in, output=out) # main sequential model model = Sequential() if not params['model_variation']=='CNN-static': model.add(Embedding(len(embedding_weights[0]), params['embedding_dim'], input_length=params['sequence_length'], weights=embedding_weights)) model.add(Dropout(params['dropout_prob'][0], input_shape=(params['sequence_length'], params['embedding_dim']))) model.add(graph) model.add(Dense(params['n_dense'])) model.add(Dropout(params['dropout_prob'][1])) model.add(Activation('relu')) model.add(Dense(output_dim=params["n_out"], init="uniform")) model.add(Activation(params['final_activation'])) logging.debug("Output CNN: %s" % str(model.output_shape)) if params['final_activation'] == 'linear': model.add(Lambda(lambda x :K.l2_normalize(x, axis=1))) return model # word2vec ARCH with LSTM
def __prepare_model(self): print('Build model...') model = Sequential() model.add(TimeDistributedDense(output_dim=self.hidden_cnt, input_dim=self.input_dim, input_length=self.input_length, activation='sigmoid')) # model.add(TimeDistributed(Dense(output_dim=self.hidden_cnt, # input_dim=self.input_dim, # input_length=self.input_length, # activation='sigmoid'))) # my modification since import error from keras.layers.core import TimeDistributedMerge # model.add(TimeDistributedMerge(mode='ave')) #comment by me ##################### my ref ######################################################### # # add a layer that returns the concatenation # # of the positive part of the input and # # the opposite of the negative part # # def antirectifier(x): # x -= K.mean(x, axis=1, keepdims=True) # x = K.l2_normalize(x, axis=1) # pos = K.relu(x) # neg = K.relu(-x) # return K.concatenate([pos, neg], axis=1) # # def antirectifier_output_shape(input_shape): # shape = list(input_shape) # assert len(shape) == 2 # only valid for 2D tensors # shape[-1] *= 2 # return tuple(shape) # # model.add(Lambda(antirectifier, output_shape=antirectifier_output_shape)) ############################################################################# model.add(Lambda(function=lambda x: K.mean(x, axis=1), output_shape=lambda shape: (shape[0],) + shape[2:])) # model.add(Dropout(0.5)) model.add(Dropout(0.93755)) model.add(Dense(self.hidden_cnt, activation='tanh')) model.add(Dense(self.output_dim, activation='softmax')) # try using different optimizers and different optimizer configs print('Compile model...') # sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) # model.compile(loss='categorical_crossentropy', optimizer=sgd) # return model ##my add adagrad = keras.optimizers.Adagrad(lr=0.01, epsilon=1e-08, decay=0.0) model.compile(loss='categorical_crossentropy', optimizer=adagrad) return model
