Python theano.tensor 模块，imatrix() 实例源码

def create_training_computation_graphs():
    x = tensor.tensor4('features')
    y = tensor.imatrix('targets')

    convnet, mlp = create_model_bricks()
    y_hat = mlp.apply(convnet.apply(x).flatten(ndim=2))
    cost = BinaryCrossEntropy().apply(y, y_hat)
    accuracy = 1 - tensor.neq(y > 0.5, y_hat > 0.5).mean()
    cg = ComputationGraph([cost, accuracy])

    # Create a graph which uses batch statistics for batch normalization
    # as well as dropout on selected variables
    bn_cg = apply_batch_normalization(cg)
    bricks_to_drop = ([convnet.layers[i] for i in (5, 11, 17)] +
                      [mlp.application_methods[1].brick])
    variables_to_drop = VariableFilter(
        roles=[OUTPUT], bricks=bricks_to_drop)(bn_cg.variables)
    bn_dropout_cg = apply_dropout(bn_cg, variables_to_drop, 0.5)

    return cg, bn_dropout_cg

def __init__(self, config):
        autoassign(locals())
        self.margin_size = config.get('margin_size', 0.2)
        self.updater = util.Adam(max_norm=config['max_norm'], lr=config['lr'])
        self.Encode = Encoder(config['size_vocab'],
                              config['size_embed'], 
                              config['size'],
                              depth=config.get('depth', 1),
                              recur_depth=config.get('recur_depth',1),
                              drop_i=config.get('drop_i', 0.75),
                              drop_s=config.get('drop_s', 0.25),
                              residual=config.get('residual', False),
                              seed=config.get('seed', 1))                              
        self.ImgEncoder  = Dense(config['size_target'], config['size'], init=eval(config.get('init_img', 'orthogonal')))
        self.inputs = [T.imatrix()]
        self.target = T.fmatrix()

def test():
    energies_var = T.tensor4('energies', dtype=theano.config.floatX)
    targets_var = T.imatrix('targets')
    masks_var = T.matrix('masks', dtype=theano.config.floatX)
    layer_input = lasagne.layers.InputLayer([2, 2, 3, 3], input_var=energies_var)
    out = lasagne.layers.get_output(layer_input)
    loss = crf_loss(out, targets_var, masks_var)
    prediction, acc = crf_accuracy(energies_var, targets_var)

    fn = theano.function([energies_var, targets_var, masks_var], [loss, prediction, acc])

    energies = np.array([[[[10, 15, 20], [5, 10, 15], [3, 2, 0]], [[5, 10, 1], [5, 10, 1], [5, 10, 1]]],
                         [[[5, 6, 7], [2, 3, 4], [2, 1, 0]], [[0, 0, 0], [0, 0, 0], [0, 0, 0]]]], dtype=np.float32)

    targets = np.array([[0, 1], [0, 2]], dtype=np.int32)

    masks = np.array([[1, 1], [1, 0]], dtype=np.float32)

    l, p, a = fn(energies, targets, masks)
    print l
    print p
    print a

def __init__(self, batch_size, emb_X, lstm_param, output_size, f1_classes):
        super().__init__(batch_size)
        self.inputs = [T.imatrix('input'), T.matrix('mask')]
        self.target = T.ivector('target')
        l = InputLayer((batch_size, None), self.inputs[0])
        l_mask = InputLayer((batch_size, None), self.inputs[1])
        l = EmbeddingLayer(l, emb_X.shape[0], emb_X.shape[1], W=emb_X)
        l = LSTMLayer(
            l, lstm_param, mask_input=l_mask, grad_clipping=100, nonlinearity=tanh,
            only_return_final=True
        )
        l = DenseLayer(l, output_size, nonlinearity=log_softmax)
        self.pred = T.exp(get_output(l, deterministic=True))
        self.loss = T.mean(categorical_crossentropy_exp(self.target, get_output(l)))
        params = get_all_params(l, trainable=True)
        self.updates = rmsprop(self.loss, params, learning_rate=0.01)
        self.metrics = {'train': [acc], 'val': [acc, f1(f1_classes)]}
        self.network = l
        self.compile()

def __init__(self, batch_size, emb_X, input_size, conv_param, lstm_param, output_size, f1_classes):
        super().__init__(batch_size)
        self.input_size = input_size
        self.conv_param = conv_param
        self.inputs = [T.imatrix('input'), T.matrix('mask')]
        self.target = T.ivector('target')
        l = InputLayer((batch_size, input_size), self.inputs[0])
        l_mask = InputLayer((batch_size, input_size + conv_param - 1), self.inputs[1])
        l = EmbeddingLayer(l, emb_X.shape[0], emb_X.shape[1], W=emb_X)
        l = DimshuffleLayer(l, (0, 2, 1))
        l = Conv1DLayer(l, 300, conv_param, pad='full', nonlinearity=rectify)
        l = DimshuffleLayer(l, (0, 2, 1))
        l = LSTMLayer(
            l, lstm_param, mask_input=l_mask, grad_clipping=100, nonlinearity=tanh,
            only_return_final=True
        )
        l = DenseLayer(l, output_size, nonlinearity=log_softmax)
        self.pred = T.exp(get_output(l, deterministic=True))
        self.loss = T.mean(categorical_crossentropy_exp(self.target, get_output(l)))
        params = get_all_params(l, trainable=True)
        self.updates = adadelta(self.loss, params)
        self.metrics = {'train': [acc], 'val': [acc, f1(f1_classes)]}
        self.network = l
        self.compile()

def set_model(self):
        argv = self.argv

        #####################
        # Network variables #
        #####################
        x = T.ftensor3()
        d = T.imatrix()

        n_in = self.init_emb.shape[1]
        n_h = argv.hidden
        n_y = self.arg_dict.size()
        reg = argv.reg

        #################
        # Build a model #
        #################
        say('\n\nMODEL:  Unit: %s  Opt: %s' % (argv.unit, argv.opt))
        self.model = Model(argv=argv, x=x, y=d, n_in=n_in, n_h=n_h, n_y=n_y, reg=reg)

def add_input(self, name, ndim=2, dtype='float'):
        if name in self.namespace:
            raise Exception('Duplicate node identifier: ' + name)
        self.namespace.add(name)
        self.input_order.append(name)
        layer = Layer() # empty layer
        if dtype == 'float':
            layer.input = ndim_tensor(ndim)
        else:
            if ndim == 2:
                layer.input = T.imatrix()
            else:
                raise Exception('Type "int" can only be used with ndim==2.')
        layer.input.name = name
        self.inputs[name] = layer
        self.output_config.append({'name':name, 'ndim':ndim, 'dtype':dtype})

def __init__(self, input_dim, proj_dim=128, 
        init='uniform', activation='sigmoid', weights=None):
        super(WordContextProduct,self).__init__()
        self.input_dim = input_dim
        self.proj_dim = proj_dim
        self.init = initializations.get(init)
        self.activation = activations.get(activation)

        self.input = T.imatrix()
        # two different embeddings for pivot word and its context
        # because p(w|c) != p(c|w)
        self.W_w = self.init((input_dim, proj_dim))
        self.W_c = self.init((input_dim, proj_dim))

        self.params = [self.W_w, self.W_c]

        if weights is not None:
            self.set_weights(weights)

def symbolic_input_variables(self):
        features = tensor.tensor3('features')
        features_mask = tensor.matrix('features_mask')
        labels = tensor.imatrix('labels')
        labels_mask = tensor.matrix('labels_mask')

        start_flag = tensor.scalar('start_flag')

        if self.use_speaker:
            speaker = tensor.imatrix('speaker_index')
        else:
            speaker = None

        if self.raw_output:
            raw_sequence = tensor.itensor3('raw_audio')
        else:
            raw_sequence = None

        return features, features_mask, labels, labels_mask, \
            speaker, start_flag, raw_sequence

def setupVariables(self):
        floatX = theano.config.floatX  # @UndefinedVariable


        # params
        self.learning_rate = T.scalar('learning_rate',dtype=floatX) 
        self.momentum = T.scalar('momentum',dtype=floatX)

        # input
        self.tvIndex = T.lscalar()  # index to a [mini]batch
        #self.tvIndex.tag.test_value = 10
        self.tvX = self.descrNet.inputVar

        # targets
        self.tvY = T.ivector('y')
        self.tvYr = T.tensor4('yr')
        self.tvPairIdx = T.imatrix('pairIdx')
        self.tvPairLabels = T.ivector('pairLabels')
        self.tvTripletIdx = T.imatrix('tripletIdx')
        self.tvTripletThresh = T.scalar('tripletThresh')
        self.tvTripletPoolIdx = T.imatrix('tripletPoolIdx')
        self.tvTripletPoolThresh = T.scalar('tripletPoolThresh')
        self.tvPosTripletPoolSize = T.iscalar('posTripletPoolSize')
        self.tvNegTripletPoolSize = T.iscalar('negTripletPoolSize')

def test_dim2(self):
        """Test the inversion of several permutations at a time"""
        # Each row of p is a different permutation to inverse
        p = imatrix()
        inv = inverse_permutation(p)
        f_inverse = function([p], inv)

        rng = numpy.random.RandomState(utt.fetch_seed())
        # Generate 10 random permutations
        p_val = numpy.asarray([rng.permutation(10) for i in range(7)],
                              dtype='int32')
        inv_val = f_inverse(p_val)

        # Check that the inverse of the inverse is the original permutation list
        assert numpy.all(f_inverse(inv_val) == p_val)
        # Check that, for each permutation,
        # permutation(inverse) == inverse(permutation) = identity
        for p_row, i_row in zip(p_val, inv_val):
            assert numpy.all(p_row[i_row] == numpy.arange(10))
            assert numpy.all(i_row[p_row] == numpy.arange(10))

def test_1_2(self):
        """Test PermuteRowElements(vector, matrix)
        Different permutations will be applied to the same input vector"""
        input = vector()
        p = imatrix()
        out = permute_row_elements(input, p)
        permute = function([input, p], out)

        rng = numpy.random.RandomState(utt.fetch_seed())
        input_val = rng.uniform(size=(5,)).astype(config.floatX)
        p_val = numpy.asarray([rng.permutation(5) for i in range(3)
            ], dtype='int32')
        out_val = permute(input_val, p_val)

        # Each row of p contains a permutation to apply to the input vector
        out_bis = numpy.asarray([input_val[p_row] for p_row in p_val])
        assert numpy.all(out_val == out_bis)

        # Verify gradient
        def permute_fixed(s_input):
            """Auxiliary op defined to get rid of gradient wrt p_val"""
            return permute_row_elements(s_input, p_val)
        utt.verify_grad(permute_fixed, [input_val])

def test_sparseblockdot(self):
        """
        Compares the numpy version of sparseblockgemv to sparse_block_dot.
        """
        b = tensor.fmatrix()
        W = tensor.ftensor4()
        h = tensor.ftensor3()
        iIdx = tensor.imatrix()
        oIdx = tensor.imatrix()

        o = sparse_block_dot(W, h, iIdx, b, oIdx)

        f = theano.function([W, h, iIdx, b, oIdx], o, mode=self.mode)

        W_val, h_val, iIdx_val, b_val, oIdx_val = \
            BlockSparse_Gemv_and_Outer.gemv_data()

        th_out = f(W_val, h_val, iIdx_val, b_val, oIdx_val)

        ref_out = BlockSparse_Gemv_and_Outer.gemv_numpy(
            b_val.take(oIdx_val, axis=0), W_val, h_val, iIdx_val, oIdx_val)

        utt.assert_allclose(ref_out, th_out)

def test_sparseblockgemv(self):
        """
        Compares the numpy and theano versions of sparseblockgemv.
        """
        b = tensor.fmatrix()
        W = tensor.ftensor4()
        h = tensor.ftensor3()
        iIdx = tensor.imatrix()
        oIdx = tensor.imatrix()

        o = self.gemv_op(b.take(oIdx, axis=0), W, h, iIdx, oIdx)

        f = theano.function([W, h, iIdx, b, oIdx], o, mode=self.mode)

        W_val, h_val, iIdx_val, b_val, oIdx_val = \
            BlockSparse_Gemv_and_Outer.gemv_data()

        th_out = f(W_val, h_val, iIdx_val, b_val, oIdx_val)
        ref_out = BlockSparse_Gemv_and_Outer.gemv_numpy(
            b_val.take(oIdx_val, axis=0), W_val, h_val, iIdx_val, oIdx_val)

        utt.assert_allclose(ref_out, th_out)

def test_sparseblockouter(self):
        o = tensor.ftensor4()
        x = tensor.ftensor3()
        y = tensor.ftensor3()
        xIdx = tensor.imatrix()
        yIdx = tensor.imatrix()

        out = self.outer_op(o, x, y, xIdx, yIdx)

        f = theano.function([o, x, y, xIdx, yIdx], out,
                            on_unused_input="warn", mode=self.mode)

        o_val, x_val, y_val, xIdx_val, yIdx_val = \
            BlockSparse_Gemv_and_Outer.outer_data()

        th_out = f(o_val, x_val, y_val, xIdx_val, yIdx_val)
        ref_out = BlockSparse_Gemv_and_Outer.outer_numpy(
            o_val, x_val, y_val, xIdx_val, yIdx_val)

        utt.assert_allclose(ref_out, th_out)

def test_shape():
    input_var = T.tensor3('input')
    target_var = T.imatrix('target')
    output_var, _, _ = memory_augmented_neural_network(
        input_var, target_var,
        batch_size=16,
        nb_class=5,
        memory_shape=(128, 40),
        controller_size=200,
        input_size=20 * 20,
        nb_reads=4)

    posterior_fn = theano.function([input_var, target_var], output_var)

    test_input = np.random.rand(16, 50, 20 * 20)
    test_target = np.random.randint(5, size=(16, 50)).astype('int32')
    test_input_invalid_batch_size = np.random.rand(16 + 1, 50, 20 * 20)
    test_input_invalid_depth = np.random.rand(16, 50, 20 * 20 - 1)
    test_output = posterior_fn(test_input, test_target)

    assert test_output.shape == (16, 50, 5)
    with pytest.raises(ValueError) as e_info:
        posterior_fn(test_input_invalid_batch_size, test_target)
    with pytest.raises(ValueError) as e_info:
        posterior_fn(test_input_invalid_depth, test_target)

def add_input(self, name, input_shape, dtype='float'):
        if name in self.namespace:
            raise Exception('Duplicate node identifier: ' + name)
        self.namespace.add(name)
        self.input_order.append(name)
        layer = Layer()  # empty layer
        layer.set_input_shape(input_shape)
        ndim = len(input_shape) + 1
        if dtype == 'float':
            layer.input = ndim_tensor(ndim)
        else:
            if ndim == 2:
                layer.input = T.imatrix()
            else:
                raise Exception('Type "int" can only be used with ndim==2 (Embedding).')
        layer.input.name = name
        self.inputs[name] = layer
        self.input_config.append({'name': name,
                                  'input_shape': input_shape,
                                  'dtype': dtype})

def __init__(self, input_dim, proj_dim=128,
                 init='uniform', activation='sigmoid', weights=None, **kwargs):

        super(WordContextProduct, self).__init__(**kwargs)
        self.input_dim = input_dim
        self.proj_dim = proj_dim
        self.init = initializations.get(init)
        self.activation = activations.get(activation)

        self.input = T.imatrix()
        # two different embeddings for pivot word and its context
        # because p(w|c) != p(c|w)
        self.W_w = self.init((input_dim, proj_dim))
        self.W_c = self.init((input_dim, proj_dim))

        self.params = [self.W_w, self.W_c]

        if weights is not None:
            self.set_weights(weights)

def train_ready(self):
        var_x = T.imatrix()
        var_y = T.imatrix()

        loss = self.l2reg(self.w, self.wdecay)+self.logp_loss(var_x,var_y)
        witems = self.w.values()
        #ave_w = sum(T.sum(item**2) for item in witems)/len(witems)

        wg = T.grad(loss, witems, disconnected_inputs = 'ignore')
        #ave_g = sum(T.sum(item**2) for item in wg) /len(wg)

        weight_up = self.upda(wg, witems, self.lrate, self.mweight, self.opt, self.gradbound)

        if not self.fix_emb:
            dicitems = self.embMatrix.values()
            dg = T.grad(loss, dicitems)
            dic_up = self.upda(dg, dicitems, self.emb_lrate, self.mweight, self.opt)
            weight_up.update(dic_up)

        up  = weight_up
        self.updatefunc = theano.function([var_x, var_y], loss, updates = up)

def add_input(self, name, ndim=2, dtype='float'):
        if name in self.namespace:
            raise Exception('Duplicate node identifier: ' + name)
        self.namespace.add(name)
        self.input_order.append(name)
        layer = Layer() # empty layer
        if dtype == 'float':
            layer.input = ndim_tensor(ndim)
        else:
            if ndim == 2:
                layer.input = T.imatrix()
            else:
                raise Exception('Type "int" can only be used with ndim==2.')
        layer.input.name = name
        self.inputs[name] = layer
        self.output_config.append({'name':name, 'ndim':ndim, 'dtype':dtype})

def __init__(self, input_dim, proj_dim=128, 
        init='uniform', activation='sigmoid', weights=None):
        super(WordContextProduct,self).__init__()
        self.input_dim = input_dim
        self.proj_dim = proj_dim
        self.init = initializations.get(init)
        self.activation = activations.get(activation)

        self.input = T.imatrix()
        # two different embeddings for pivot word and its context
        # because p(w|c) != p(c|w)
        self.W_w = self.init((input_dim, proj_dim))
        self.W_c = self.init((input_dim, proj_dim))

        self.params = [self.W_w, self.W_c]

        if weights is not None:
            self.set_weights(weights)

def build_train_func(rank=0, **kwargs):
    print("rank: {} Building model".format(rank))
    resnet = build_resnet()

    print("Building training function")
    x = T.ftensor4('x')
    y = T.imatrix('y')

    prob = L.get_output(resnet['prob'], x, deterministic=False)
    loss = T.nnet.categorical_crossentropy(prob, y.flatten()).mean()
    params = L.get_all_params(resnet.values(), trainable=True)

    sgd_updates = updates.sgd(loss, params, learning_rate=1e-4)

    # make a function to compute and store the raw gradient
    f_train = theano.function(inputs=[x, y],
                              outputs=loss,  # (assumes this is an avg)
                              updates=sgd_updates)

    return f_train, "original"

def create(self):
        self._input_name = 'text'
        self._output_name = 'output'

        self.add_input(
                name=self._input_name, 
                input_shape=(self._config.max_input_time_steps, self._config.input_dim,))
        self.inputs['text'].input = T.imatrix()
        self.add_node(Embedding(
                self._config.input_dim, 
                self._config.textual_embedding_dim, 
                mask_zero=True), 
                name='embedding', input='text')
        self.add_node(
                self._config.recurrent_encoder(
                    self._config.hidden_state_dim, 
                    return_sequences=False,
                    go_backwards=self._config.go_backwards),
                name='recurrent', input='embedding') 
        self.add_node(Dropout(0.5), name='dropout', input='recurrent')
        self.add_node(Dense(self._config.output_dim), name='dense', input='dropout')
        self.add_node(Activation('softmax'), name='softmax', input='dense')
        self.add_output(name=self._output_name, input='softmax')

def ndim_itensor(ndim, name=None):
    if ndim == 2:
        return T.imatrix(name)
    elif ndim == 3:
        return T.itensor3(name)
    elif ndim == 4:
        return T.itensor4(name)
    return T.imatrix(name)


# dot-product

def ndim_itensor(ndim, name=None):
    if ndim == 2:
        return T.imatrix(name)
    elif ndim == 3:
        return T.itensor3(name)
    elif ndim == 4:
        return T.itensor4(name)
    return T.imatrix(name)


# dot-product

def __init__(self, rng, embeddings, char_embeddings, hiddensize, char_hiddensize, embedding_dim, char_embedding_dim, window_size, num_tags, dic_size, dropout_rate = 0.7):
        self.rng = rng
        self.inputX = T.imatrix('inputX') # a sentence, shape (T * window_size)
        self.inputX_chars = T.itensor3('inputX_chars') # a sentence, shape (T * max numbe of chars in a word)
        self.inputY = T.ivector('inputY') # tags of a sentence
        self.is_train = T.iscalar('is_train')

        self.new_theta = T.fmatrix('new_theta')

        self.dropout_rate = dropout_rate
        self.nhidden = hiddensize
        self.char_nhidden = char_hiddensize # for now set the number of hidden units the same
        self.embedding_dim = embedding_dim
        self.char_embedding_dim = char_embedding_dim
        self.window_size = window_size
        self.n_classes = num_tags
        self.dic_size = dic_size

        # for testing in compling
        self.inputX.tag.test_value = np.ones((10, window_size)).astype(np.int32)
        self.inputX_chars.tag.test_value = np.ones((10, window_size, 8)).astype(np.int32)
        self.inputY.tag.test_value = np.ones(10).astype(np.int32)

        self.Embeddings = theano.shared(value = embeddings, name = "Embeddings", borrow = True)
        self.Char_Embeddings = theano.shared(value = char_embeddings, name = "Char_Embeddings", borrow = True)

        # word embeddings
        self.inputW = self.Embeddings[self.inputX]

        # char embeddings
        self.inputC = self.Char_Embeddings[self.inputX_chars].dimshuffle([2, 0, 1, 3])

        self.params = [self.Embeddings, self.Char_Embeddings]

def __init__(self, config):
        autoassign(locals())
        self.margin_size = config.get('margin_size', 0.2)
        self.updater = util.Adam(max_norm=config['max_norm'], lr=config['lr'])
        self.Encode = Encoder(config['size_vocab'],
                              config['size_embed'])
        self.ImgEncoder  = Dense(config['size_target'], config['size_embed'], init=eval(config.get('init_img', 'orthogonal')))
        self.inputs = [T.imatrix()]
        self.target = T.fmatrix()
        self.config['margin'] = self.config.get('margin', False)

def __init__(
        self, batch_size, emb_X, input_size, conv_param, dense_params, output_size, max_norm, f1_classes
    ):
        super().__init__(batch_size)
        self.input_size = input_size
        self.inputs = [T.imatrix('input')]
        self.target = T.ivector('target')
        l = InputLayer((self.batch_size, input_size), self.inputs[0])
        l = EmbeddingLayer(l, emb_X.shape[0], emb_X.shape[1], W=emb_X)
        l = DimshuffleLayer(l, (0, 2, 1))
        l_convs = []
        for filter_size in conv_param[1]:
            l_cur = Conv1DLayer(l, conv_param[0], filter_size, pad='full', nonlinearity=rectify)
            l_cur = MaxPool1DLayer(l_cur, input_size + filter_size - 1, ignore_border=True)
            l_cur = FlattenLayer(l_cur)
            l_convs.append(l_cur)
        l = ConcatLayer(l_convs)
        l = DropoutLayer(l)
        for dense_param in dense_params:
            l = DenseLayer(l, dense_param, nonlinearity=rectify)
            self.constraints[l.W] = lambda u, v: norm_constraint(v, max_norm)
            l = DropoutLayer(l)
        l = DenseLayer(l, output_size, nonlinearity=identity)
        self.constraints[l.W] = lambda u, v: norm_constraint(v, max_norm)
        l = SelfInteractionLayer(l, nonlinearity=log_softmax)
        self.pred = T.exp(get_output(l, deterministic=True))
        self.loss = T.mean(categorical_crossentropy_exp(self.target, get_output(l)))
        params = get_all_params(l, trainable=True)
        self.updates = adadelta(self.loss, params)
        self.metrics = {'train': [acc], 'val': [acc, f1(f1_classes)]}
        self.network = l
        self.compile()

def __init__(self, batch_size, emb_X, input_size, output_size, static_mode, f1_classes):
        super().__init__(batch_size)
        self.input_size = input_size
        self.inputs = [T.imatrix('input')]
        self.target = T.ivector('target')
        l = InputLayer((batch_size, input_size), self.inputs[0])
        l = EmbeddingLayer(l, emb_X.shape[0], emb_X.shape[1], W=emb_X)
        if static_mode == 0:
            self.constraints[l.W] = lambda u, v: u
        l = DimshuffleLayer(l, (0, 2, 1))
        conv_params = [(256, 7, 3), (256, 7, 3), (256, 3, None), (256, 3, None), (256, 3, None), (256, 3, 3)]
        for num_filters, filter_size, k in conv_params:
            l = Conv1DLayer(l, num_filters, filter_size, nonlinearity=rectify)
            if k is not None:
                l = MaxPool1DLayer(l, k, ignore_border=False)
        l = FlattenLayer(l)
        dense_params = [(1024, 1), (1024, 20)]
        for num_units, max_norm in dense_params:
            l = DenseLayer(l, num_units, nonlinearity=rectify)
            if max_norm:
                self.constraints[l.W] = lambda u, v: norm_constraint(v, max_norm)
            l = DropoutLayer(l)
        l = DenseLayer(l, output_size, nonlinearity=log_softmax)
        self.pred = T.exp(get_output(l, deterministic=True))
        self.loss = T.mean(categorical_crossentropy_exp(self.target, get_output(l)))
        params = get_all_params(l, trainable=True)
        self.updates = adadelta(self.loss, params)
        self.metrics = {'train': [acc], 'val': [acc, f1(f1_classes)]}
        self.network = l
        self.compile()

def __init__(self, number_words, num_hidden, seq_length, mb_size):
        self.mb_size = mb_size
        x = T.imatrix()
        target = T.ivector()
        word_embeddings = theano.shared(np.random.normal(size = ((number_words, 1, num_hidden))).astype('float32'))
        feature_lst = []
        for i in range(0, seq_length):
            feature = word_embeddings[x[:,i]]
            feature_lst.append(feature)
        features = T.concatenate(feature_lst, 1)

        #example x sequence_position x feature
        #inp = InputLayer(shape = (seq_length, mb_size, num_hidden), input_var = features)
        l_lstm_1 = LSTMLayer((seq_length, mb_size, num_hidden), num_units = num_hidden, nonlinearity = lasagne.nonlinearities.tanh)
        l_lstm_2 = LSTMLayer((seq_length, mb_size, num_hidden), num_units = num_hidden, nonlinearity = lasagne.nonlinearities.tanh)

        #minibatch x sequence x feature
        final_out = T.mean(l_lstm_2.get_output_for([l_lstm_1.get_output_for([features])]), axis = 1)

        #final_out = T.mean(features, axis = 1)
        h_out = DenseLayer((mb_size, num_hidden), num_units = 1, nonlinearity=None)
        h_out_value = h_out.get_output_for(final_out)
        classification = T.nnet.sigmoid(h_out_value)
        self.loss = T.mean(T.nnet.binary_crossentropy(output = classification.flatten(), target = target))
        self.params = lasagne.layers.get_all_params(h_out,trainable=True) + [word_embeddings] + lasagne.layers.get_all_params(l_lstm_1, trainable = True) + lasagne.layers.get_all_params(l_lstm_2, trainable = True)
        updates = lasagne.updates.adam(self.loss, self.params)
        self.train_func = theano.function(inputs = [x, target], outputs = {'l' : self.loss, 'c' : classification}, updates = updates)
        self.evaluate_func = theano.function(inputs = [x], outputs = {'c' : classification})

def __init__(self, input_dim, output_dim, init='uniform',
        W_regularizer=None, activity_regularizer=None, W_constraint=None,
        mask_zero=False, weights=None):

        super(Embedding,self).__init__()
        self.init = initializations.get(init)
        self.input_dim = input_dim
        self.output_dim = output_dim

        self.input = T.imatrix()
        self.W = self.init((self.input_dim, self.output_dim))
        self.mask_zero = mask_zero

        self.params = [self.W]
        self.constraints = [W_constraint]

        self.regularizers = []
        if W_regularizer:
            W_regularizer.set_param(self.W)
            self.regularizers.append(W_regularizer)
        if activity_regularizer:
            activity_regularizer.set_layer(self)
            self.regularizers.append(activity_regularizer)

        if weights is not None:
            self.set_weights(weights)

def raw_inputs(self):
        seq = tensor.imatrix('rseq')
        feat = tensor.tensor3('rfeat')
        h0_ = tensor.tensor3('rh0')
        big_h0_ = tensor.tensor3('rbigh0')
        res_ = tensor.scalar('rscalar')
        mask_ = tensor.matrix('rmask')

        return seq, feat, h0_, big_h0_, res_, mask_

def getting_generation_functions(sequences, h0, big_h0, reset, features):


    big_frame_level_generate_fn = theano.function(
        [sequences, big_h0, reset, features],
        big_frame_level_rnn(sequences, big_h0, reset, features)[0:2],
        on_unused_input='warn'
    )

    big_frame_level_outputs = T.matrix('big_frame_level_outputs')

    frame_level_generate_fn = theano.function(
        [sequences, big_frame_level_outputs, h0, reset],
        frame_level_rnn(sequences, big_frame_level_outputs.dimshuffle(0,'x',1), h0, reset),
        on_unused_input='warn'
    )

    frame_level_outputs = T.matrix('frame_level_outputs')
    prev_samples        = T.imatrix('prev_samples')
    sample_level_generate_fn = theano.function(
        [frame_level_outputs, prev_samples],
        lib.ops.softmax_and_sample(
            sample_level_predictor(
                frame_level_outputs,
                prev_samples
            ),
            #temperature=TEMPERATURE,
            temperature=1.0,
        ),
        on_unused_input='warn'
    )
    return big_frame_level_generate_fn, frame_level_generate_fn, sample_level_generate_fn


# # Sampling at audio sample level

def test_lnlstm_get_emb_output():
    hid_size = 10
    inp_size = 10
    out_size = 40
    n_batches = 23
    seqlen = 47
    l_in = InputLayer((n_batches, seqlen), input_var=T.imatrix('input_var'), name="l_in")
    l_emb = EmbeddingLayer(l_in, inp_size, out_size, name="l_emb")
    l_lstm = LNLSTMLayer(l_emb, hid_size, name="l_lstm")

    emb_output = lasagne.layers.get_output(l_emb)
    output = lasagne.layers.get_output(l_lstm)
    output_for = l_lstm.get_output_for([emb_output])

def test_lstm_get_emb_output():
    hid_size = 10
    inp_size = 10
    out_size = 40
    l_in = InputLayer((None, None), input_var=T.imatrix())
    l_emb = EmbeddingLayer(l_in, inp_size, out_size)
    l_lstm = LSTMLayer(l_emb, hid_size)

    emb_output = lasagne.layers.get_output(l_emb)
    output = lasagne.layers.get_output(l_lstm)
    output_for = l_lstm.get_output_for([emb_output])

def __init__(self):
        self.model = []
        theano.config.floatX = "float32"
        self.X = T.tensor4()
        self.Y = T.imatrix()#Y(batchsize,vector)
        # self.X = T.tensor4()
        # self.Y = T.imatrix()
        # self.X = T.matrix()
        # self.Y = T.imatrix()
        self.train_data = []

def __init__(self,n_input, n_hidden, n_output,
                 cell='gru', optimizer='sgd',p=0.1,
                 q_w=None,k=10,sampling='nce'):
        self.x = T.imatrix('batched_sequence_x')  # n_batch, maxlen
        self.x_mask = T.fmatrix('x_mask')
        self.y = T.imatrix('batched_sequence_y')
        self.y_mask = T.fmatrix('y_mask')
        # negy is the negative sampling for nce shape (len(y),k)
        self.negy = T.itensor3('negy')

        self.n_input=n_input
        self.n_hidden=n_hidden
        self.n_output=n_output

        self.k=k
        self.sampling=sampling
        self.optimizer=optimizer
        self.cell=cell
        self.optimizer=optimizer
        self.p=p
        self.is_train = T.iscalar('is_train')
        self.rng = RandomStreams(1234)


        init_Embd = np.asarray(
            np.random.uniform(low=-np.sqrt(1. / n_output), high=np.sqrt(1. / n_output), size=(n_output, n_input)),
            dtype=theano.config.floatX)

        self.E = theano.shared(value=init_Embd, name='word_embedding', borrow=True)



        self.q_w = theano.shared(value=q_w, name='vocabulary distribution', borrow=True)


        self.build()

def ndim_itensor(ndim, name=None):
    if ndim == 2:
        return T.imatrix(name)
    elif ndim == 3:
        return T.itensor3(name)
    elif ndim == 4:
        return T.itensor4(name)
    return T.imatrix(name)


# dot-product

def setupSetMacroBatchSubset(self):

        if isinstance(self.tvsData_x,list):
            data_block = [T.tensor4('data_block_{}'.format(i)) for i in range(len(self.tvsData_x))]
            data_updates = [(dx,T.set_subtensor(dx[:db.shape[0]], db)) for (dx,db) in zip(self.tvsData_x,data_block)]
        else:
            data_block = T.tensor4('data_block')
            data_updates = [(self.tvsData_x, T.set_subtensor(self.tvsData_x[:data_block.shape[0]], data_block))]
        self.tfSetMacroBatchSubsetData = theano.function(inputs=[data_block],updates=data_updates)

        if self.cfgParams.use_labels:
            y_block = T.ivector('y_block')
            y_updates = [(self.tvsData_y, T.set_subtensor(self.tvsData_y[:y_block.shape[0]], y_block))]
            self.tfSetMacroBatchSubsetY = theano.function(inputs=[y_block],updates=y_updates)

        if self.cfgParams.use_regtargets:
            yr_block = T.vector('yr_block')
            yr_updates = [(self.tvsData_yr, T.set_subtensor(self.tvsData_yr[:yr_block.shape[0]], yr_block))]
            self.tfSetMacroBatchSubsetYR = theano.function(inputs=[yr_block],updates=yr_updates)

        if self.cfgParams.use_pairs:
            pairIdx_block = T.imatrix('pairIdx_block')
            pairLabels_block = T.ivector('pairLabels_block')
            pair_updates = [( self.tvsData_pairIdx, T.set_subtensor(self.tvsData_pairIdx[:pairIdx_block.shape[0]], pairIdx_block) ),
                            ( self.tvsData_pairLabels, T.set_subtensor(self.tvsData_pairLabels[:pairLabels_block.shape[0]], pairLabels_block) )]
            self.tfSetMacroBatchSubsetPairs = theano.function(inputs=[pairIdx_block,pairLabels_block],updates=pair_updates)

        if self.cfgParams.use_triplets:
            tripletIdx_block = T.imatrix('tripletIdx_block')
            triplets_updates = [ (self.tvsData_tripletIdx, T.set_subtensor(self.tvsData_tripletIdx[:tripletIdx_block.shape[0]], tripletIdx_block)) ]
            self.tfSetMacroBatchSubsetTriplets = theano.function(inputs=[tripletIdx_block],updates=triplets_updates)

        if self.cfgParams.use_tripletPools:            
            tripletPoolIdx_block = T.imatrix('tripletPoolIdx_block')
            tripletPools_updates = [ (self.tvsData_tripletPoolIdx, T.set_subtensor(self.tvsData_tripletPoolIdx[:tripletPoolIdx_block.shape[0]], tripletPoolIdx_block)) ]
            self.tfSetMacroBatchSubsetTripletPools = theano.function(inputs=[tripletPoolIdx_block],updates=tripletPools_updates)

def make_node(self, x, x2, x3, x4):
        # check that the theano version has support for __props__.
        # This next line looks like it has a typo,
        # but it's actually a way to detect the theano version
        # is sufficiently recent to support the use of __props__.
        assert hasattr(self, '_props'), "Your version of theano is too old to support __props__."
        x = tensor.as_tensor_variable(x)
        x2 = tensor.as_tensor_variable(x2)
        x3 = tensor.as_tensor_variable(x3)
        x4 = tensor.as_tensor_variable(x4)
        return theano.Apply(self, [x, x2, x3, x4], [tensor.fvector().type(), tensor.imatrix().type()])

def __build__model__(self):
        x = T.imatrix('x')  # first sentence
        x_mask = T.fmatrix("x_mask")
        y = T.ivector('y')  # label

        # for compatibility, input's shape is (sen_len, batch_size)
        # for cnn convolution, the input shape should be (batch_size, 1, sen_len, embed_dim)
        # embedding encoder and decoder inputs with two embedding layers
        embedding = self.word_embedding_layer.embedding(x.flatten())
        embedding = embedding.reshape([x.shape[0], 1, x.shape[1], self.embed_dim])
        embedding = embedding.dimshuffle(2, 1, 0, 3)
        # embedding = embedding.reshape([input.shape[0], input.shape[1], self.embed_dim])

        conv_outs = [conv_layer.get_output(embedding) for conv_layer in self.conv_layers]
        conv_hidden = T.concatenate(conv_outs, axis=1)
        # conv_hidden = theano.printing.Print("Conv Hidden")(conv_hidden)

        final_hidden_states = self.mlp_layers.get_output(conv_hidden)

        train_weights, _ = self.predict_layer.predict(final_hidden_states, drop=self.drop, train=True)
        # calculate cross entropy
        cost = T.sum(T.nnet.categorical_crossentropy(train_weights, y.flatten()))

        cost += self.predict_layer.get_l2_cost() * 0.01

        learning_rate = T.scalar('learning_rate')
        decay = T.scalar('decay')

        grads, updates = model_utils.rms_prop(cost, self.param_names, self.params, learning_rate, decay)

        # Assign functions
        self.loss = theano.function([x, x_mask, y], cost, on_unused_input='ignore')
        self.bptt = theano.function([x, x_mask, y], grads, on_unused_input='ignore')
        self.sgd_step = theano.function(
            [x, x_mask, y, learning_rate, decay],
            updates=updates, on_unused_input='ignore')

        # for test
        weights, predictions = self.predict_layer.predict(final_hidden_states, drop=self.drop, train=False)
        self.weights = theano.function([x, x_mask], weights, on_unused_input='ignore')
        self.predictions = theano.function([x, x_mask], predictions, on_unused_input='ignore')

def __build__model__(self):
        input = T.imatrix('input')  # first sentence
        input_mask = T.fmatrix('input_mask')  # mask
        target = T.ivector('target')  # label

        # embedding encoder and decoder inputs with two embedding layers
        encoding = self.word_embedding_layer.embedding(input.flatten())
        encoding = encoding.reshape([input.shape[0], input.shape[1], self.embed_dim])
        # encodes and decodes
        encoding, _ = self.encoder.encode(encoding, input_mask)

        final_hidden_states = encoding[-1, :, :] # shape(batch_size, embedding)
        train_weights, _ = self.predict_layer.predict(final_hidden_states, drop=self.drop, train=True)

        # calculate cross entropy
        cost = T.sum(T.nnet.categorical_crossentropy(train_weights, target.flatten()))

        cost += self.predict_layer.get_l2_cost() * 0.01

        learning_rate = T.scalar('learning_rate')
        decay = T.scalar('decay')

        grads, updates = model_utils.rms_prop(cost, self.param_names, self.params, learning_rate, decay)

        # Assign functions
        self.loss = theano.function([input, input_mask, target], cost, on_unused_input='ignore')
        self.bptt = theano.function([input, input_mask, target], grads, on_unused_input='ignore')
        self.sgd_step = theano.function(
            [input, input_mask, target, learning_rate, decay],
            updates=updates, on_unused_input='ignore')

        # for test
        weights, predictions = self.predict_layer.predict(final_hidden_states, drop=self.drop, train=False)
        self.weights = theano.function([input, input_mask], weights, on_unused_input='ignore')
        self.predictions = theano.function([input, input_mask], predictions, on_unused_input='ignore')

def symbolic_input_variables(self):
        data = tensor.tensor3('features')
        data_mask = tensor.matrix('features_mask')
        context = tensor.imatrix('transcripts')
        context_mask = tensor.matrix('transcripts_mask')
        start_flag = tensor.scalar('start_flag')

        return data, data_mask, context, context_mask, start_flag

def setup_encode(self):

        # dimensions: (batch, time, 12)
        chord_types = T.btensor3()
        # dimensions: (batch, time)
        chord_roots = T.imatrix()
        # dimensions: (batch, time)
        relative_posns = [T.imatrix() for _ in self.encodings]
        # dimesions: (batch, time, output_data)
        encoded_melodies = [T.btensor3() for _ in self.encodings]
        n_batch, n_time = chord_roots.shape

        all_activations = []
        for encoding, enc_lstmstack, encoded_melody, relative_pos in zip(self.encodings, self.enc_lstmstacks, encoded_melodies, relative_posns):
            activations = enc_lstmstack.do_preprocess_scan( timestep=T.tile(T.arange(n_time), (n_batch,1)) ,
                                                        relative_position=relative_pos,
                                                        cur_chord_type=chord_types,
                                                        cur_chord_root=chord_roots,
                                                        cur_input=encoded_melody,
                                                        deterministic_dropout=True )
            all_activations.append(activations)
        reduced_activations = functools.reduce((lambda x,y: x+y), all_activations)
        strengths, vects = self.qman.get_strengths_and_vects(reduced_activations)

        self.encode_fun = theano.function(
            inputs=[chord_types, chord_roots] + relative_posns + encoded_melodies,
            outputs=[strengths, vects],
            allow_input_downcast=True,
            mode=(NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True) if self.nanguard else None))

def setup_decode(self):

        # dimensions: (batch, time, 12)
        chord_types = T.btensor3()
        # dimensions: (batch, time)
        chord_roots = T.imatrix()
        # dimensions: (batch, time)
        feat_strengths = T.fmatrix()
        # dimensions: (batch, time, feature_size)
        feat_vects = T.ftensor3()
        n_batch, n_time = chord_roots.shape

        features = QueueManager.queue_transform(feat_strengths, feat_vects)

        specs = [lstmstack.prepare_sample_scan(  start_pos=T.alloc(np.array(encoding.STARTING_POSITION, np.int32), (n_batch)),
                                                    start_out=T.tile(encoding.initial_encoded_form(), (n_batch,1)),
                                                    timestep=T.tile(T.arange(n_time), (n_batch,1)),
                                                    cur_chord_type=chord_types,
                                                    cur_chord_root=chord_roots,
                                                    cur_feature=features,
                                                    deterministic_dropout=True )
                    for lstmstack, encoding in zip(self.dec_lstmstacks, self.encodings)]

        updates, all_chosen, all_probs, indiv_probs = helper_generate_from_spec(specs, self.dec_lstmstacks, self.encodings, self.srng, n_batch, n_time, self.bounds)

        self.decode_fun = theano.function(
            inputs=[chord_roots, chord_types, feat_strengths, feat_vects],
            updates=updates,
            outputs=all_chosen,
            allow_input_downcast=True,
            mode=(NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True) if self.nanguard else None))

        self.decode_visualize_fun = theano.function(
            inputs=[chord_roots, chord_types, feat_strengths, feat_vects],
            updates=updates,
            outputs=[all_chosen, all_probs] + indiv_probs + [features],
            allow_input_downcast=True,
            mode=(NanGuardMode(nan_is_error=True, inf_is_error=True, big_is_error=True) if self.nanguard else None))