Python keras.backend 模块，floatx() 实例源码

def add_boundary_energy(x, b_start=None, b_end=None, mask=None):
    '''Given the observations x, it adds the start boundary energy b_start (resp.
    end boundary energy b_end on the start (resp. end) elements and multiplies
    the mask.'''
    if mask is None:
        if b_start is not None:
            x = K.concatenate([x[:, :1, :] + b_start, x[:, 1:, :]], axis=1)
        if b_end is not None:
            x = K.concatenate([x[:, :-1, :], x[:, -1:, :] + b_end], axis=1)
    else:
        mask = K.cast(mask, K.floatx())
        mask = K.expand_dims(mask, 2)
        x *= mask
        if b_start is not None:
            mask_r = K.concatenate([K.zeros_like(mask[:, :1]), mask[:, :-1]], axis=1)
            start_mask = K.cast(K.greater(mask, mask_r), K.floatx())
            x = x + start_mask * b_start
        if b_end is not None:
            mask_l = K.concatenate([mask[:, 1:], K.zeros_like(mask[:, -1:])], axis=1)
            end_mask = K.cast(K.greater(mask, mask_l), K.floatx())
            x = x + end_mask * b_end
    return x

def call(self, x, mask=None):
        eij = dot_product(x, self.W)

        if self.bias:
            eij += self.b

        eij = K.tanh(eij)

        a = K.exp(eij)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ? to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1)

def call(self, x, mask=None):
        uit = dot_product(x, self.W)

        if self.bias:
            uit += self.b

        uit = K.tanh(uit)
        ait = K.dot(uit, self.u)

        a = K.exp(ait)

        # apply mask after the exp. will be re-normalized next
        if mask is not None:
            # Cast the mask to floatX to avoid float64 upcasting in theano
            a *= K.cast(mask, K.floatx())

        # in some cases especially in the early stages of training the sum may be almost zero
        # and this results in NaN's. A workaround is to add a very small positive number ? to the sum.
        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())
        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())

        a = K.expand_dims(a)
        weighted_input = x * a
        return K.sum(weighted_input, axis=1)

def build(self, input_shape):
        assert len(input_shape) == 2
        input_dim = input_shape[1]
        #self.input_spec = InputSpec(dtype=K.floatx(), shape=(None, input_dim))
        self.input_spec = InputSpec(ndim=2, axes={1: input_dim})

        self.W = self.add_weight(
                shape=(input_dim, self.output_dim),
                initializer=self.W_initializer,
                name='SparseFullyConnected_W',
                regularizer=self.W_regularizer,
                constraint=self.W_constraint)
        self.b = self.add_weight(
                shape=(self.output_dim,),
                initializer=self.b_initializer,
                name='SparseFullyConnected_b',
                regularizer=self.b_regularizer,
                constraint=self.b_constraint)


        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights
        #self.built = True
        #super(SparseFullyConnectedLayer, self).build(input_shape)

def call(self, x, mask=None):
        # x: [..., time_steps, features]
        # ut = [..., time_steps, attention_dims]
        ut = K.dot(x, self.kernel)
        if self.use_bias:
            ut = K.bias_add(ut, self.bias)

        ut = K.tanh(ut)
        if self.use_context:
            ut = ut * self.context_kernel

        # Collapse `attention_dims` to 1. This indicates the weight for each time_step.
        ut = K.sum(ut, axis=-1, keepdims=True)

        # Convert those weights into a distribution but along time axis.
        # i.e., sum of alphas along `time_steps` axis should be 1.
        self.at = _softmax(ut, dim=1)
        if mask is not None:
            self.at *= K.cast(K.expand_dims(mask, -1), K.floatx())

        # Weighted sum along `time_steps` axis.
        return K.sum(x * self.at, axis=-2)

def fit(self, x, augment=False, rounds=1, seed=None):
    x = np.asarray(x, dtype=K.floatx())

    if x.ndim != 2:
      raise ValueError('Input to `.fit()` should have rank 2. '
        'Got array with shape: ' + str(x.shape))

    if seed is not None:
      np.random.seed(seed)

    x = np.copy(x)
    if augment:
      ax = np.zeros(tuple([rounds * x.shape[0]] + list(x.shape)[1:]), dtype=K.floatx())
      for r in range(rounds):
        for i in range(x.shape[0]):
          ax[i + r * x.shape[0]] = self.random_transform(x[i])
      x = ax

# @Class: Iterator
# @Description:
#   Abstract base class for Music data iterators.
#   n: Integer, total number of samples in the dataset to loop over.
#   batch_size: Integer, size of a batch.
#   shuffle: Boolean, whether to shuffle the data between epochs.
#   seed: Random seeding for data shuffling.

def __init__(self, x, y, music_data_generator,
    batch_size=32, shuffle=False, seed=None):
    if y is not None and len(x) != len(y):
      raise ValueError('X (music tensor) and y (labels) '
        'should have the same length. '
        'Found: X.shape = %s, y.shape = %s' %
        (np.asarray(x).shape, np.asarray(y).shape))

    self.x = np.asarray(x, dtype=K.floatx())
    if self.x.ndim != 2:
      raise ValueError('Input data in `NumpyArrayIterator` '
        'should have rank 2. You passed an array '
        'with shape', self.x.shape)

    if y is not None:
      self.y = np.asarray(y)
    else:
        self.y = None
    self.music_data_generator = music_data_generator
    super(NumpyArrayIterator, self).__init__(x.shape[0], batch_size, shuffle, seed)

def next(self):
    # Keeps under lock only the mechanism which advances
    # the indexing of each batch.
    with self.lock:
      index_array, current_index, current_batch_size = next(self.index_generator)

    # The transformation of images is not under thread lock
    # so it can be done in parallel
    melspec_size = [128,128]
    batch_x = np.zeros(tuple([current_batch_size] + melspec_size), dtype=K.floatx())
    for i, j in enumerate(index_array):
      x = self.x[j]
      x = self.music_data_generator.random_transform(x.astype(K.floatx()))
      batch_x[i] = x

    if self.y is None:
      return batch_x
    batch_y = self.y[index_array]
    return batch_x, batch_y

def call(self, x, mask=None):
        y = K.dot(x, self.att_W)
        if not self.activation:
            if K.backend() == 'theano':
                weights = K.theano.tensor.tensordot(self.att_v, y, axes=[0, 2])
            elif K.backend() == 'tensorflow':
                weights = K.tensorflow.python.ops.math_ops.tensordot(self.att_v, y, axes=[0, 2])
        elif self.activation == 'tanh':
            if K.backend() == 'theano':
                weights = K.theano.tensor.tensordot(self.att_v, K.tanh(y), axes=[0, 2])
            elif K.backend() == 'tensorflow':
                weights = K.tensorflow.python.ops.math_ops.tensordot(self.att_v, K.tanh(y), axes=[0, 2])
        weights = K.softmax(weights)
        out = x * K.permute_dimensions(K.repeat(weights, x.shape[2]), [0, 2, 1])
        if self.op == 'attsum':
            out = out.sum(axis=1)
        elif self.op == 'attmean':
            out = out.sum(axis=1) / mask.sum(axis=1, keepdims=True)
        return K.cast(out, K.floatx())

def test_sub_pixel_upscaling():
    num_samples = 2
    num_row = 16
    num_col = 16
    input_dtype = K.floatx()

    for scale_factor in [2, 3, 4]:
        input_data = np.random.random((num_samples, 4 * (scale_factor ** 2), num_row, num_col))
        input_data = input_data.astype(input_dtype)

        if K.image_data_format() == 'channels_last':
            input_data = input_data.transpose((0, 2, 3, 1))

        input_tensor = K.variable(input_data)
        expected_output = K.eval(KC.depth_to_space(input_tensor,
                                                   scale=scale_factor))

        layer_test(convolutional.SubPixelUpscaling,
                   kwargs={'scale_factor': scale_factor},
                   input_data=input_data,
                   expected_output=expected_output,
                   expected_output_dtype=K.floatx())

def _forward(x, reduce_step, initial_states, U, mask=None):
    '''Forward recurrence of the linear chain crf.'''

    def _forward_step(energy_matrix_t, states):
        alpha_tm1 = states[-1]
        new_states = reduce_step(K.expand_dims(alpha_tm1, 2) + energy_matrix_t)
        return new_states[0], new_states

    U_shared = K.expand_dims(K.expand_dims(U, 0), 0)

    if mask is not None:
        mask = K.cast(mask, K.floatx())
        mask_U = K.expand_dims(K.expand_dims(mask[:, :-1] * mask[:, 1:], 2), 3)
        U_shared = U_shared * mask_U

    inputs = K.expand_dims(x[:, 1:, :], 2) + U_shared
    inputs = K.concatenate([inputs, K.zeros_like(inputs[:, -1:, :, :])], axis=1)

    last, values, _ = K.rnn(_forward_step, inputs, initial_states)
    return last, values

def load_img(img_path, target_size):
    """
    loads the image the same way darknet does, processes it and returns it (as array).
    uses PIL, like keras.preprocessing.image module.
    This loads image in RGB format.
    """
    from PIL import Image as pil_image
    import keras.backend as K
    import numpy as np
    img = pil_image.open(img_path)
    # TODO: check format and convert to RGB
    #resize
    x = np.asarray(img, dtype=K.floatx())/255.0
    #print(x[0,0,0], x[1,0,0], x[0,1,0], x[1,1,0], img.mode)
    x = letterbox_image(x, target_size)
    return x

def call(self, x, mask=None):
        # x should be an output and a target
        assert len(x) == 2

        losses = _per_sample_loss(self.loss, mask, x)
        if self.fast:
            grads = K.sqrt(sum([
                K.sum(K.square(g), axis=1)
                for g in K.gradients(losses, self.parameter_list)
            ]))
        else:
            nb_samples = K.shape(losses)[0]
            grads = K.map_fn(
                lambda i: self._grad_norm(losses[i]),
                K.arange(0, nb_samples),
                dtype=K.floatx()
            )

        return K.reshape(grads, (-1, 1))

def call(self, x, mask=None):
        # x should be an output and a target
        assert len(x) == 2

        losses = _per_sample_loss(self.loss, mask, x)
        if self.fast:
            grads = K.sqrt(sum([
                K.sum(K.square(g), axis=1)
                for g in K.gradients(losses, self.parameter_list)
            ]))
        else:
            nb_samples = K.shape(losses)[0]
            grads = K.map_fn(
                lambda i: self._grad_norm(losses[i]),
                K.arange(0, nb_samples),
                dtype=K.floatx()
            )

        return K.reshape(grads, (-1, 1))

def call(self, x, mask=None):
        # x should be an output and a target
        assert len(x) == 2

        losses = _per_sample_loss(self.loss, mask, x)
        if self.fast:
            grads = K.sqrt(sum([
                K.sum(K.square(g), axis=1)
                for g in K.gradients(losses, self.parameter_list)
            ]))
        else:
            nb_samples = K.shape(losses)[0]
            grads = K.map_fn(
                lambda i: self._grad_norm(losses[i]),
                K.arange(0, nb_samples),
                dtype=K.floatx()
            )

        return K.reshape(grads, (-1, 1))

def build(self, input_shape):
        alpha_shape = input_shape[self.axis]

        self.alpha = self.init((alpha_shape,),
                                name='alpha_pos'.format(self.name))
        self.rho = K.variable(self.power_init * np.ones(alpha_shape),
                                 name='rho_pos'.format(self.name))

        if self.fit:
            self.trainable_weights = [self.alpha, self.rho]

        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=input_shape)]

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights

def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        self.input_dim = input_dim
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     ndim='2+')]

        self.W = self.add_weight((input_dim, self.output_dim),
                                 initializer=self.init,
                                 name='{}_W'.format(self.name),
                                 regularizer=self.W_regularizer,
                                 constraint=self.W_constraint)
        if self.bias:
            self.b = self.add_weight((self.output_dim,),
                                     initializer='zero',
                                     name='{}_b'.format(self.name),
                                     regularizer=self.b_regularizer,
                                     constraint=self.b_constraint)
        else:
            self.b = None

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights
        self.built = True

def build(self, input_shape):
        assert len(input_shape) >= 2
        input_dim = input_shape[-1]
        self.input_dim = input_dim
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     ndim='2+')]

        self.W = self.add_weight((input_dim, input_dim),
                                 initializer=self.init,
                                 name='{}_W'.format(self.name),
                                 regularizer=self.W_regularizer,
                                 constraint=self.W_constraint)
        if self.bias:
            self.b = self.add_weight((input_dim,),
                                     initializer='zero',
                                     name='{}_b'.format(self.name),
                                     regularizer=self.b_regularizer,
                                     constraint=self.b_constraint)
        else:
            self.b = None

        if self.initial_weights is not None:
            self.set_weights(self.initial_weights)
            del self.initial_weights
        self.built = True

def add_boundary_energy(x, b_start=None, b_end=None, mask=None):
    '''Given the observations x, it adds the start boundary energy b_start (resp.
    end boundary energy b_end on the start (resp. end) elements and multiplies
    the mask.'''
    if mask is None:
        if b_start is not None:
            x = K.concatenate([x[:, :1, :] + b_start, x[:, 1:, :]], axis=1)
        if b_end is not None:
            x = K.concatenate([x[:, :-1, :], x[:, -1:, :] + b_end], axis=1)
    else:
        mask = K.cast(mask, K.floatx())
        mask = K.expand_dims(mask, 2)
        x *= mask
        if b_start is not None:
            mask_r = K.concatenate([K.zeros_like(mask[:, :1]), mask[:, :-1]], axis=1)
            start_mask = K.cast(K.greater(mask, mask_r), K.floatx())
            x = x + start_mask * b_start
        if b_end is not None:
            mask_l = K.concatenate([mask[:, 1:], K.zeros_like(mask[:, -1:])], axis=1)
            end_mask = K.cast(K.greater(mask, mask_l), K.floatx())
            x = x + end_mask * b_end
    return x

def _forward(x, reduce_step, initial_states, U, mask=None):
    '''Forward recurrence of the linear chain crf.'''

    def _forward_step(energy_matrix_t, states):
        alpha_tm1 = states[-1]
        new_states = reduce_step(K.expand_dims(alpha_tm1, 2) + energy_matrix_t)
        return new_states[0], new_states

    U_shared = K.expand_dims(K.expand_dims(U, 0), 0)

    if mask is not None:
        mask = K.cast(mask, K.floatx())
        mask_U = K.expand_dims(K.expand_dims(mask[:, :-1] * mask[:, 1:], 2), 3)
        U_shared = U_shared * mask_U

    inputs = K.expand_dims(x[:, 1:, :], 2) + U_shared
    inputs = K.concatenate([inputs, K.zeros_like(inputs[:, -1:, :, :])], axis=1)

    last, values, _ = K.rnn(_forward_step, inputs, initial_states)
    return last, values

def call(self, x, mask=None):
        # computes a probability distribution over the timesteps
        # uses 'max trick' for numerical stability
        # reshape is done to avoid issue with Tensorflow
        # and 1-dimensional weights
        logits = K.dot(x, self.W)
        x_shape = K.shape(x)
        logits = K.reshape(logits, (x_shape[0], x_shape[1]))
        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))

        # masked timesteps have zero weight
        if mask is not None:
            mask = K.cast(mask, K.floatx())
            ai = ai * mask
        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())
        weighted_input = x * K.expand_dims(att_weights)
        result = K.sum(weighted_input, axis=1)
        if self.return_attention:
            return [result, att_weights]
        return result

def get_sample_weights(y, class_weights=None):
    """Compute sample weights for model training.

    Computes sample weights given  a vector of output labels `y`. Sets weights
    of samples without label (`CPG_NAN`) to zero.

    Parameters
    ----------
    y: :class:`numpy.ndarray`
        1d numpy array of output labels.
    class_weights: dict
        Weight of output classes, e.g. methylation states.

    Returns
    -------
    :class:`numpy.ndarray`
        Sample weights of size `y`.
    """
    y = y[:]
    sample_weights = np.ones(y.shape, dtype=K.floatx())
    sample_weights[y == dat.CPG_NAN] = K.epsilon()
    if class_weights is not None:
        for cla, weight in class_weights.items():
            sample_weights[y == cla] = weight
    return sample_weights

def contingency_table(y, z):
    """Compute contingency table."""
    y = K.round(y)
    z = K.round(z)

    def count_matches(a, b):
        tmp = K.concatenate([a, b])
        return K.sum(K.cast(K.all(tmp, -1), K.floatx()))

    ones = K.ones_like(y)
    zeros = K.zeros_like(y)
    y_ones = K.equal(y, ones)
    y_zeros = K.equal(y, zeros)
    z_ones = K.equal(z, ones)
    z_zeros = K.equal(z, zeros)

    tp = count_matches(y_ones, z_ones)
    tn = count_matches(y_zeros, z_zeros)
    fp = count_matches(y_zeros, z_ones)
    fn = count_matches(y_ones, z_zeros)

    return (tp, tn, fp, fn)

def call(self, x, mask=None):
        def image_expand(tensor):
            return K.expand_dims(K.expand_dims(tensor, -1), -1)

        def batch_image_expand(tensor):
            return image_expand(K.expand_dims(tensor, 0))

        hw = K.cast(x.shape[2] * x.shape[3], K.floatx())
        mu = K.sum(x, [-1, -2]) / hw
        mu_vec = image_expand(mu) 
        sig2 = K.sum(K.square(x - mu_vec), [-1, -2]) / hw
        y = (x - mu_vec) / (K.sqrt(image_expand(sig2)) + K.epsilon())

        scale = batch_image_expand(self.scale)
        shift = batch_image_expand(self.shift)
        return scale*y + shift 
#       else:
#           raise NotImplemented("Please complete `CycGAN/layers/padding.py` to run on backend {}.".format(K.backend()))

def build(self, input_shape):
        assert len(input_shape) == 2

        input_dim = input_shape[1]
        self.input_spec = [InputSpec(dtype=K.floatx(),
                                     shape=(None, input_dim))]

        self.W = self.add_weight(shape=(self.nb_kernels, input_dim, self.kernel_dim),
            initializer=self.init,
            name='kernel',
            regularizer=self.W_regularizer,
            trainable=True,
            constraint=self.W_constraint)

        # Set built to true.
        super(MinibatchDiscrimination, self).build(input_shape)

def test_set_floatx(self):
        """
        Make sure that changes to the global floatx are effectively
        taken into account by the backend.
        """
        # Keep track of the old value
        old_floatx = floatx()

        set_floatx('float16')
        var = variable([10])
        check_dtype(var, 'float16')

        set_floatx('float64')
        var = variable([10])
        check_dtype(var, 'float64')

        # Restore old value
        set_floatx(old_floatx)

def build(self, input_shape):

        # construct the clockwork structures
        # basically: every n units the period changes;
        # `period` is for flaggin this; `mask` is for enforcing it
        n = self.output_dim // len(self.period_spec)
        mask = np.zeros((self.output_dim, self.output_dim), K.floatx())
        period = np.zeros((self.output_dim,), np.int16)
        for i, t in enumerate(self.period_spec):
            mask[i*n:(i+1)*n, i*n:] = 1
            period[i*n:(i+1)*n] = t

        self.mask = K.variable(mask, name='clockword_mask')
        self.period = K.variable(period, dtype='int16', name='clockwork_period')

        super(ClockworkRNN, self).build(input_shape)

        self.U = self.U * self.mask  # old implementation did this at run time...

        # simple rnn initializes the wrong size self.states
        # we want to also keep the time step in the state.
        if self.stateful:
            self.reset_states()
        else:
            self.states = [None, None]

def call(self, inputs, mask=None):
        # In case the target shape is not fully defined,
        # we need access to the shape of x.
        # solution:
        # 1) rely on x._keras_shape
        # 2) fallback: K.int_shape
        target_shape = self.target_shape
        target_mask_shape = self.target_mask_shape
        if -1 in target_shape:
            # target shape not fully defined
            input_shape = None
            try:
                input_shape = K.int_shape(inputs)
            except TypeError:
                pass
            if input_shape is not None:
                target_shape = self.compute_output_shape(input_shape)[1:]

        _result = K.reshape(inputs, (-1,) + target_shape)
        reshaped_mask = K.reshape(mask, (-1,) + target_mask_shape + (1,))
        result = _result * K.cast(reshaped_mask, K.floatx())
        return result

def __init__(self, maxlen, d_L, d_C, d_D, V_C):
    """
    maxlen = maximum input/output word size
    d_L    = language model hidden state (= context vector) size
    d_C    = character features (input embedding vector size)
    d_D    = decoder hidden state h size
    V_C    = character vocabulary
    """
    # extend embeddings to treat zero values as zeros vectors (for y_0 = 0)
    # but don't do any masking
    class CharEmb(Embedding):
      def call(self, x, mask=None):
        y = super(CharEmb, self).call(x)
        return y * K.cast(K.expand_dims(x, -1), K.floatx())

    c       = Input(shape=(d_L,), name='c')
    y_tm1   = Input(shape=(maxlen,), name='y_tm1', dtype='int32')

    ye_tm1  = CharEmb(V_C.size + 1, d_C)(y_tm1)
    h       = DecoderGRU(d_D, return_sequences=True)([ye_tm1, c])
    s       = Maxout(d_C)([h, ye_tm1, RepeatVector(maxlen)(c)])
    s       = Dropout(.2)(s)
    c_I     = ProjectionOverTime(V_C.size)(s)

    super(W2C, self).__init__(input=[c, y_tm1], output=c_I, name='W2C')

def build(self, input_shape):
    self.input_spec = [InputSpec(dtype=K.floatx(),
                                 shape=(None, input_shape[0][1], input_shape[0][2])),
                       InputSpec(dtype=K.floatx(),
                                 shape=(None, input_shape[1][1], input_shape[1][2])),
                       InputSpec(dtype=K.floatx(),
                                 shape=(None, input_shape[2][1], input_shape[2][2]))]

    self.W_h = self.init((self.nb_feature, input_shape[0][2], self.output_dim),
                         name='{}_W_h'.format(self.name))

    self.W_y = self.init((self.nb_feature, input_shape[1][2], self.output_dim),
                         name='{}_W_y'.format(self.name))

    self.W_c = self.init((self.nb_feature, input_shape[2][2], self.output_dim),
                         name='{}_W_c'.format(self.name))

    trainable = [self.W_h, self.W_y, self.W_c]

    if self.bias:
      self.b = K.zeros((self.nb_feature, self.output_dim),
                       name='{}_b'.format(self.name))
      self.trainable_weights = trainable + [self.b]
    else:
      self.trainable_weights = trainable

def add_boundary_energy(x, b_start=None, b_end=None, mask=None):
    """Given the observations x, it adds the start boundary energy b_start (resp.
    end boundary energy b_end on the start (resp. end) elements and multiplies
    the mask."""
    if mask is None:
        if b_start is not None:
            x = K.concatenate([x[:, :1, :] + b_start, x[:, 1:, :]], axis=1)
        if b_end is not None:
            x = K.concatenate([x[:, :-1, :], x[:, -1:, :] + b_end], axis=1)
    else:
        mask = K.cast(mask, K.floatx())
        mask = K.expand_dims(mask, 2)
        x *= mask
        if b_start is not None:
            mask_r = K.concatenate([K.zeros_like(mask[:, :1]), mask[:, :-1]], axis=1)
            start_mask = K.cast(K.greater(mask, mask_r), K.floatx())
            x = x + start_mask * b_start
        if b_end is not None:
            mask_l = K.concatenate([mask[:, 1:], K.zeros_like(mask[:, -1:])], axis=1)
            end_mask = K.cast(K.greater(mask, mask_l), K.floatx())
            x = x + end_mask * b_end
    return x

def _forward(x, reduce_step, initial_states, U, mask=None):
    """Forward recurrence of the linear chain crf."""

    def _forward_step(energy_matrix_t, states):
        alpha_tm1 = states[-1]
        new_states = reduce_step(K.expand_dims(alpha_tm1, 2) + energy_matrix_t)
        return new_states[0], new_states

    U_shared = K.expand_dims(K.expand_dims(U, 0), 0)

    if mask is not None:
        mask = K.cast(mask, K.floatx())
        mask_U = K.expand_dims(K.expand_dims(mask[:, :-1] * mask[:, 1:], 2), 3)
        U_shared = U_shared * mask_U

    inputs = K.expand_dims(x[:, 1:, :], 2) + U_shared
    inputs = K.concatenate([inputs, K.zeros_like(inputs[:, -1:, :, :])], axis=1)

    last, values, _ = K.rnn(_forward_step, inputs, initial_states)
    return last, values

def test_set_floatx(self):
        """
        Make sure that changes to the global floatx are effectively
        taken into account by the backend.
        """
        # Keep track of the old value
        old_floatx = floatx()

        set_floatx('float16')
        var = variable([10])
        check_dtype(var, 'float16')

        set_floatx('float64')
        var = variable([10])
        check_dtype(var, 'float64')

        # Restore old value
        set_floatx(old_floatx)

def test_grams_loss(self):
        input = np.zeros((1, 3, 4, 4))
        iter = 0
        for i in range(input.shape[1]):
            for j in range(input.shape[2]):
                for k in range(input.shape[3]):
                    input[0][i][j][k] = iter
                    iter += 1
        input = input.astype(K.floatx())


        x = K.placeholder(input.shape, name='x')
        gram_mat = grams(x)
        loss = frobenius_error(gram_mat, np.ones((1, 3, 3)))
        get_loss = K.function([x], [loss])

        error = get_loss([input])[0]
        true_error = 60344.299382716

        self.assertEqual(np.round(error.item(0)), np.round(true_error))

def load_mnist():
    '''
    returns mnist_data
    '''
    # input image dimensions
    img_rows, img_cols = 28, 28

    # the data, shuffled and split between train and test sets
    (x_train, y_train), (x_test, y_test) = mnist.load_data()

    if k.image_data_format() == 'channels_first':
        x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
        input_shape = (1, img_rows, img_cols)
    else:
        x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
        input_shape = (img_rows, img_cols, 1)

    x_train = x_train.astype(k.floatx())
    x_train *= 0.96/255
    x_train += 0.02
    return input_shape, x_train

def step(self, input_energy_t, states, return_logZ=True):
        # not in the following  `prev_target_val` has shape = (B, F)
        # where B = batch_size, F = output feature dim
        # Note: `i` is of float32, due to the behavior of `K.rnn`
        prev_target_val, i, chain_energy = states[:3]
        t = K.cast(i[0, 0], dtype='int32')
        if len(states) > 3:
            if K.backend() == 'theano':
                m = states[3][:, t:(t + 2)]
            else:
                m = K.tf.slice(states[3], [0, t], [-1, 2])
            input_energy_t = input_energy_t * K.expand_dims(m[:, 0])
            chain_energy = chain_energy * K.expand_dims(K.expand_dims(m[:, 0] * m[:, 1]))  # (1, F, F)*(B, 1, 1) -> (B, F, F)
        if return_logZ:
            energy = chain_energy + K.expand_dims(input_energy_t - prev_target_val, 2)  # shapes: (1, B, F) + (B, F, 1) -> (B, F, F)
            new_target_val = K.logsumexp(-energy, 1)  # shapes: (B, F)
            return new_target_val, [new_target_val, i + 1]
        else:
            energy = chain_energy + K.expand_dims(input_energy_t + prev_target_val, 2)
            min_energy = K.min(energy, 1)
            argmin_table = K.cast(K.argmin(energy, 1), K.floatx())  # cast for tf-version `K.rnn`
            return argmin_table, [min_energy, i + 1]

def call(self, inputs, mask=None):
        t = inputs
        timegate = K.abs(self.timegate)
        period = timegate[0]
        shift = timegate[1]
        r_on = timegate[2]

        phi = ((t - shift) % period) / period
        # K.switch not consistent between Theano and Tensorflow backend,
        # so write explicitly.
        # TODO check if still the case
        up = K.cast(K.less(phi, r_on * 0.5), K.floatx()) * 2 * phi / r_on
        mid = (
            K.cast(K.less(phi, r_on), K.floatx()) *
            K.cast(K.greater(phi, r_on * 0.5), K.floatx()) *
            (2 - (2 * phi / r_on))
        )
        end = K.cast(K.greater(phi, r_on * 0.5), K.floatx()) * self.alpha * phi
        k = up + mid + end

        return k

def prepare_data(imagenet_dir, batch_size, num_threads):
    # setup image loading
    dataset = dataset_factory.get_dataset('imagenet', 'validation', imagenet_dir)
    provider = slim.dataset_data_provider.DatasetDataProvider(dataset,
                                                              shuffle=False,
                                                              common_queue_capacity=batch_size * 5,
                                                              common_queue_min=batch_size)
    image, label = provider.get(['image', 'label'])

    # preprocess images and split into batches
    preprocess_input = preprocessing_factory.get_preprocessing('inception_resnet_v2',
                                                               is_training=False)
    image = preprocess_input(image, 299, 299)
    images, labels = tf.train.batch([image, label],
                                    batch_size=batch_size,
                                    num_threads=num_threads,
                                    capacity=batch_size * 5)

    # Keras label is different from TF
    labels = labels - 1  # remove the "background class"
    labels = K.cast(K.expand_dims(labels, -1), K.floatx())  # Keras labels are 2D float tensors
    return images, labels, dataset.num_samples

def build(self, input_shape):

        ### construct the clockwork structures
        ### basically: every n units the period changes;
        ### `period` is for flaggin this; `mask` is for enforcing it
        n = self.output_dim // len(self.period_spec)
        mask = np.zeros((self.output_dim, self.output_dim), K.floatx())
        period = np.zeros((self.output_dim,), np.int16)
        for i, t in enumerate(self.period_spec):
            mask[i*n:(i+1)*n, i*n:] = 1
            period[i*n:(i+1)*n] = t
        self.mask = K.variable(mask, name='clockword_mask')
        self.period = K.variable(period, dtype='int16', name='clockwork_period')

        super(ClockworkRNN, self).build(input_shape)

        self.U = self.U * self.mask  ### old implementation did this at run time... 

        ### simple rnn initializes the wrong size self.states
        ### we want to also keep the time step in the state. 
        if self.stateful:
            self.reset_states()
        else:
            self.states = [None, None]

def call(self, x, mask=None):
        if self.mode == 'maximum_likelihood':
            # draw maximum likelihood sample from Bernoulli distribution
            #    x* = argmax_x p(x) = 1         if p(x=1) >= 0.5
            #                         0         otherwise
            return K.round(x)
        elif self.mode == 'random':
            # draw random sample from Bernoulli distribution
            #    x* = x ~ p(x) = 1              if p(x=1) > uniform(0, 1)
            #                    0              otherwise
            #return self.srng.binomial(size=x.shape, n=1, p=x, dtype=K.floatx())
            return K.random_binomial(x.shape, p=x, dtype=K.floatx())
        elif self.mode == 'mean_field':
            # draw mean-field approximation sample from Bernoulli distribution
            #    x* = E[p(x)] = E[Bern(x; p)] = p
            return x
        elif self.mode == 'nrlu':
            return nrlu(x)
        else:
            raise NotImplementedError('Unknown sample mode!')

def gen_cosine_amp(amp=100, period=1000, x0=0, xn=50000, step=1, k=0.0001):
    """Generates an absolute cosine time series with the amplitude
    exponentially decreasing

    Arguments:
        amp: amplitude of the cosine function
        period: period of the cosine function
        x0: initial x of the time series
        xn: final x of the time series
        step: step of the time series discretization
        k: exponential rate
    """
    cos = np.zeros(((xn - x0) * step, 1, 1), dtype=K.floatx())
    for i in range(len(cos)):
        idx = x0 + i * step
        cos[i, 0, 0] = amp * np.cos(2 * np.pi * idx / period)
        cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
    return cos

def kde_entropy(output, var):
    # Kernel density estimate of entropy, in nats

    dims = K.cast(K.shape(output)[1], K.floatx() ) 
    N    = K.cast(K.shape(output)[0], K.floatx() )

    normconst = (dims/2.0)*K.log(2*np.pi*var)

    # get dists matrix
    x2 = K.expand_dims(K.sum(K.square(output), axis=1), 1)
    dists = x2 + K.transpose(x2) - 2*K.dot(output, K.transpose(output))
    dists = dists / (2*var)

    lprobs = logsumexp(-dists, axis=1) - K.log(N) - normconst
    h = -K.mean(lprobs)

    return h

def img_to_array(img, dim_ordering='default'):
    """Converts a PIL Image instance to a Numpy array.

    # Arguments
        img: PIL Image instance.
        dim_ordering: Image data format.

    # Returns
        A 3D Numpy array.

    # Raises
        ValueError: if invalid `img` or `dim_ordering` is passed.
    """
    if dim_ordering == 'default':
        dim_ordering = K.image_dim_ordering()
    if dim_ordering not in {'th', 'tf'}:
        raise ValueError('Unknown dim_ordering: ', dim_ordering)
    # Numpy array x has format (height, width, channel)
    # or (channel, height, width)
    # but original PIL image has format (width, height, channel)
    x = np.asarray(img, dtype=K.floatx())
    if len(x.shape) == 3:
        if dim_ordering == 'th':
            x = x.transpose(2, 0, 1)
    elif len(x.shape) == 2:
        if dim_ordering == 'th':
            x = x.reshape((1, x.shape[0], x.shape[1]))
        else:
            x = x.reshape((x.shape[0], x.shape[1], 1))
    else:
        raise ValueError('Unsupported image shape: ', x.shape)
    return x

def __init__(self, x, y, image_data_generator,
                 batch_size=32, shuffle=False, seed=None,
                 dim_ordering='default',
                 save_to_dir=None, save_prefix='', save_format='jpeg'):
        if y is not None and len(x) != len(y):
            raise ValueError('X (images tensor) and y (labels) '
                             'should have the same length. '
                             'Found: X.shape = %s, y.shape = %s' %
                             (np.asarray(x).shape, np.asarray(y).shape))
        if dim_ordering == 'default':
            dim_ordering = K.image_dim_ordering()
        self.x = np.asarray(x, dtype=K.floatx())
        #if self.x.ndim != 4:  # orig
        #if self.x.ndim != 5:
        if self.x.ndim < 4 or self.x.ndim > 5:
            # below it wqas 4
            raise ValueError('Input data in `NumpyArrayIterator` '
                             'should have rank 5 (image) or rank 4(mask). You passed an array '
                             'with shape', self.x.shape)
        #channels_axis = 3 if dim_ordering == 'tf' else 1 ### orig
        channels_axis = 3 if dim_ordering == 'tf' else 1
        if self.x.shape[channels_axis] not in {1, 3, 4, 5, 6, 7, 8, 9, 11}:      ## ANDRE mod, allow also 5 and 7, 9 and 11
            raise ValueError('NumpyArrayIterator is set to use the '
                             'dimension ordering convention "' + dim_ordering + '" '
                             '(channels on axis ' + str(channels_axis) + '), i.e. expected '
                             'either 1, 3 or 4 channels on axis ' + str(channels_axis) + '. '
                             'However, it was passed an array with shape ' + str(self.x.shape) +
                             ' (' + str(self.x.shape[channels_axis]) + ' channels).')
        if y is not None:
            self.y = np.asarray(y)
        else:
            self.y = None
        self.image_data_generator = image_data_generator
        self.dim_ordering = dim_ordering
        self.save_to_dir = save_to_dir
        self.save_prefix = save_prefix
        self.save_format = save_format
        super(NumpyArrayIterator, self).__init__(x.shape[0], batch_size, shuffle, seed)

def next(self):
        # for python 2.x.
        # Keeps under lock only the mechanism which advances
        # the indexing of each batch
        # see http://anandology.com/blog/using-iterators-and-generators/
        with self.lock:
            index_array, current_index, current_batch_size = next(self.index_generator)
        # The transformation of images is not under thread lock
        # so it can be done in parallel
        batch_x = np.zeros(tuple([current_batch_size] + list(self.x.shape)[1:]), dtype=K.floatx())
        for i, j in enumerate(index_array):
            x = self.x[j]
            x = self.image_data_generator.random_transform(x.astype(K.floatx()))
            x = self.image_data_generator.standardize(x)
            batch_x[i] = x
        if self.save_to_dir:
            for i in range(current_batch_size):
                img = array_to_img(batch_x[i], self.dim_ordering, scale=True)
                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
                                                                  index=current_index + i,
                                                                  hash=np.random.randint(1e4),
                                                                  format=self.save_format)
                img.save(os.path.join(self.save_to_dir, fname))
        if self.y is None:
            return batch_x
        batch_y = self.y[index_array]
        return batch_x, batch_y

def next(self):
        with self.lock:
            index_array, current_index, current_batch_size = next(self.index_generator)
        # The transformation of images is not under thread lock
        # so it can be done in parallel
        batch_x = np.zeros((current_batch_size,) + self.image_shape, dtype=K.floatx())
        grayscale = self.color_mode == 'grayscale'
        # build batch of image data
        for i, j in enumerate(index_array):
            fname = self.filenames[j]
            img = load_img(os.path.join(self.directory, fname),
                           grayscale=grayscale,
                           target_size=self.target_size)
            x = img_to_array(img, dim_ordering=self.dim_ordering)
            x = self.image_data_generator.random_transform(x)
            x = self.image_data_generator.standardize(x)
            batch_x[i] = x
        # optionally save augmented images to disk for debugging purposes
        if self.save_to_dir:
            for i in range(current_batch_size):
                img = array_to_img(batch_x[i], self.dim_ordering, scale=True)
                fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
                                                                  index=current_index + i,
                                                                  hash=np.random.randint(1e4),
                                                                  format=self.save_format)
                img.save(os.path.join(self.save_to_dir, fname))
        # build batch of labels
        if self.class_mode == 'sparse':
            batch_y = self.classes[index_array]
        elif self.class_mode == 'binary':
            batch_y = self.classes[index_array].astype(K.floatx())
        elif self.class_mode == 'categorical':
            batch_y = np.zeros((len(batch_x), self.nb_class), dtype=K.floatx())
            for i, label in enumerate(self.classes[index_array]):
                batch_y[i, label] = 1.
        else:
            return batch_x
        return batch_x, batch_y

def call(self, inputs):
        inputs_shape = K.int_shape(inputs)
        channel_axis = len(inputs_shape) - 1
        masked_tensor = self.compute_mask(inputs)
        masked_tensor = K.expand_dims(masked_tensor)
        masked_tensor = K.repeat_elements(
            masked_tensor,
            inputs_shape[channel_axis],
            channel_axis
        )
        return inputs * K.cast(masked_tensor, K.floatx())

def compute_mask(self, inputs, mask=None):
        channel_axis = K.ndim(inputs) - 1
        mask_tensor = K.cast(mask, K.floatx())
        mask_tensor = K.expand_dims(mask_tensor)
        mask_output = self.layer._pooling_function(
            mask_tensor,
            self.layer.pool_size,
            self.layer.strides,
            self.layer.padding,
            self.layer.data_format,
        )
        mask_output = K.sum(mask_output, axis=channel_axis)
        next_mask_tensor = K.not_equal(mask_output, 0.0)
        return next_mask_tensor