Python tensorflow.python.ops.array_ops 模块，zeros() 实例源码

def zero_fast_weights(self, batch_size, dtype):
    """Return zero-filled fast_weights tensor(s).

    Args:
      batch_size: int, float, or unit Tensor representing the batch size.
      dtype: the data type to use for the state.

    Returns:
      A zero filled fast_weights of shape [batch_size, state_size, state_size]
    """
    state_size = self.state_size

    zeros = array_ops.zeros(
        array_ops.pack([batch_size, state_size, state_size]), dtype=dtype)
    zeros.set_shape([None, state_size, state_size])

    return zeros

def initial_alignments(self, batch_size, dtype):
    """Creates the initial alignment values for the `AttentionWrapper` class.

    This is important for AttentionMechanisms that use the previous alignment
    to calculate the alignment at the next time step (e.g. monotonic attention).

    The default behavior is to return a tensor of all zeros.

    Args:
      batch_size: `int32` scalar, the batch_size.
      dtype: The `dtype`.

    Returns:
      A `dtype` tensor shaped `[batch_size, alignments_size]`
      (`alignments_size` is the values' `max_time`).
    """
    max_time = self._alignments_size
    return _zero_state_tensors(max_time, batch_size, dtype)

def _create_local(name, shape, collections=None, validate_shape=True,
                  dtype=dtypes.float32):
    """Creates a new local variable.
    Args:
        name: The name of the new or existing variable.
        shape: Shape of the new or existing variable.
        collections: A list of collection names to which the Variable will be added.
        validate_shape: Whether to validate the shape of the variable.
        dtype: Data type of the variables.
    Returns:
        The created variable.
    """
    # Make sure local variables are added to tf.GraphKeys.LOCAL_VARIABLES
    collections = list(collections or [])
    collections += [ops.GraphKeys.LOCAL_VARIABLES]
    return variables.Variable(
            initial_value=array_ops.zeros(shape, dtype=dtype),
            name=name,
            trainable=False,
            collections=collections,
            validate_shape=validate_shape)

def zeros_like(x, dtype=None, name=None):
      """Instantiates an all-zeros variable of the same shape as another tensor.

      Arguments:
          x: Keras variable or Keras tensor.
          dtype: String, dtype of returned Keras variable.
               None uses the dtype of x.
          name: String, name for the variable to create.

      Returns:
          A Keras variable with the shape of x filled with zeros.

      Example:
      ```python
          >>> from keras import backend as K
          >>> kvar = K.variable(np.random.random((2,3)))
          >>> kvar_zeros = K.zeros_like(kvar)
          >>> K.eval(kvar_zeros)
          array([[ 0.,  0.,  0.],
                 [ 0.,  0.,  0.]], dtype=float32)

def count_params(x):
      """Returns the number of scalars in a Keras variable.

      Arguments:
          x: Keras variable.

      Returns:
          Integer, the number of scalars in `x`.

      Example:
      ```python
          >>> kvar = K.zeros((2,3))
          >>> K.count_params(kvar)
          6
          >>> K.eval(kvar)
          array([[ 0.,  0.,  0.],
                 [ 0.,  0.,  0.]], dtype=float32)

def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        shapes = [K.int_shape(p) for p in params]
        accumulators = [K.zeros(shape) for shape in shapes]
        self.weights = accumulators
        self.updates = []

        lr = self.lr
        if self.initial_decay > 0:
          lr *= (1. / (1. + self.decay * self.iterations))
          self.updates.append(K.update_add(self.iterations, 1))

        for p, g, a in zip(params, grads, accumulators):
          # update accumulator
          new_a = self.rho * a + (1. - self.rho) * K.square(g)
          self.updates.append(K.update(a, new_a))
          new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)

          # apply constraints
          if p in constraints:
            c = constraints[p]
            new_p = c(new_p)
          self.updates.append(K.update(p, new_p))
        return self.updates

def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        shapes = [K.int_shape(p) for p in params]
        accumulators = [K.zeros(shape) for shape in shapes]
        self.weights = accumulators
        self.updates = []

        lr = self.lr
        if self.initial_decay > 0:
          lr *= (1. / (1. + self.decay * self.iterations))
          self.updates.append(K.update_add(self.iterations, 1))

        for p, g, a in zip(params, grads, accumulators):
          new_a = a + K.square(g)  # update accumulator
          self.updates.append(K.update(a, new_a))
          new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
          # apply constraints
          if p in constraints:
            c = constraints[p]
            new_p = c(new_p)
          self.updates.append(K.update(p, new_p))
        return self.updates

def get_test_data(train_samples,
                      test_samples,
                      input_shape,
                      num_classes):
      """Generates test data to train a model on.

      Arguments:
        train_samples: Integer, how many training samples to generate.
        test_samples: Integer, how many test samples to generate.
        input_shape: Tuple of integers, shape of the inputs.
        num_classes: Integer, number of classes for the data and targets.
          Only relevant if `classification=True`.

      Returns:
        A tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
      """
      num_sample = train_samples + test_samples
      templates = 2 * num_classes * np.random.random((num_classes,) + input_shape)
      y = np.random.randint(0, num_classes, size=(num_sample,))
      x = np.zeros((num_sample,) + input_shape)
      for i in range(num_sample):
        x[i] = templates[y[i]] + np.random.normal(loc=0, scale=1., size=input_shape)
      return ((x[:train_samples], y[:train_samples]),
              (x[train_samples:], y[train_samples:]))

def to_categorical(y, num_classes=None):
          """Converts a class vector (integers) to binary class matrix.

          E.g. for use with categorical_crossentropy.

          Arguments:
              y: class vector to be converted into a matrix
                  (integers from 0 to num_classes).
              num_classes: total number of classes.

          Returns:
              A binary matrix representation of the input.
          """
          y = np.array(y, dtype='int').ravel()
          if not num_classes:
            num_classes = np.max(y) + 1
          n = y.shape[0]
          categorical = np.zeros((n, num_classes))
          categorical[np.arange(n), y] = 1
          return categorical

def __init__(self,
                       alpha_initializer='zeros',
                       alpha_regularizer=None,
                       alpha_constraint=None,
                       shared_axes=None,
                       **kwargs):
            super(PReLU, self).__init__(**kwargs)
            self.supports_masking = True
            self.alpha_initializer = initializers.get(alpha_initializer)
            self.alpha_regularizer = regularizers.get(alpha_regularizer)
            self.alpha_constraint = constraints.get(alpha_constraint)
            if shared_axes is None:
              self.shared_axes = None
            elif not isinstance(shared_axes, (list, tuple)):
              self.shared_axes = [shared_axes]
            else:
              self.shared_axes = list(shared_axes)

def _create_local(name, shape, collections=None, validate_shape=True,
                  dtype=dtypes.float32):
    """Creates a new local variable.
    Args:
        name: The name of the new or existing variable.
        shape: Shape of the new or existing variable.
        collections: A list of collection names to which the Variable will be added.
        validate_shape: Whether to validate the shape of the variable.
        dtype: Data type of the variables.
    Returns:
        The created variable.
    """
    # Make sure local variables are added to tf.GraphKeys.LOCAL_VARIABLES
    collections = list(collections or [])
    collections += [ops.GraphKeys.LOCAL_VARIABLES]
    return variables.Variable(
            initial_value=array_ops.zeros(shape, dtype=dtype),
            name=name,
            trainable=False,
            collections=collections,
            validate_shape=validate_shape)

def _create_local(name, shape, collections=None, validate_shape=True,
                  dtype=dtypes.float32):
  """Creates a new local variable.

  Args:
    name: The name of the new or existing variable.
    shape: Shape of the new or existing variable.
    collections: A list of collection names to which the Variable will be added.
    validate_shape: Whether to validate the shape of the variable.
    dtype: Data type of the variables.

  Returns:
    The created variable.
  """
  # Make sure local variables are added to tf.GraphKeys.LOCAL_VARIABLES
  collections = list(collections or [])
  collections += [ops.GraphKeys.LOCAL_VARIABLES]
  return variables.Variable(
      initial_value=array_ops.zeros(shape, dtype=dtype),
      name=name,
      trainable=False,
      collections=collections,
      validate_shape=validate_shape)

def _forward(self, x):
    # Pad the last dim with a zeros vector. We need this because it lets us
    # infer the scale in the inverse function.
    y = array_ops.expand_dims(x, dim=-1) if self._static_event_ndims == 0 else x
    ndims = (y.get_shape().ndims if y.get_shape().ndims is not None
             else array_ops.rank(y))
    y = array_ops.pad(y, paddings=array_ops.concat(0, (
        array_ops.zeros((ndims - 1, 2), dtype=dtypes.int32),
        [[0, 1]])))

    # Set shape hints.
    if x.get_shape().ndims is not None:
      shape = x.get_shape().as_list()
      if self._static_event_ndims == 0:
        shape += [2]
      elif shape[-1] is not None:
        shape[-1] += 1
      shape = tensor_shape.TensorShape(shape)
      y.get_shape().assert_is_compatible_with(shape)
      y.set_shape(shape)

    # Since we only support event_ndims in [0, 1] and we do padding, we always
    # reduce over the last dimension, i.e., dim=-1 (which is the default).
    return nn_ops.softmax(y)

def _create_local(name, shape, collections=None, validate_shape=True,
                  dtype=dtypes.float32):
    """Creates a new local variable.
    Args:
        name: The name of the new or existing variable.
        shape: Shape of the new or existing variable.
        collections: A list of collection names to which the Variable will be added.
        validate_shape: Whether to validate the shape of the variable.
        dtype: Data type of the variables.
    Returns:
        The created variable.
    """
    # Make sure local variables are added to tf.GraphKeys.LOCAL_VARIABLES
    collections = list(collections or [])
    collections += [ops.GraphKeys.LOCAL_VARIABLES]
    return variables.Variable(
            initial_value=array_ops.zeros(shape, dtype=dtype),
            name=name,
            trainable=False,
            collections=collections,
            validate_shape=validate_shape)

def zero_fast_weights(self, batch_size, dtype):
    """Return zero-filled fast_weights tensor(s).

    Args:
      batch_size: int, float, or unit Tensor representing the batch size.
      dtype: the data type to use for the state.

    Returns:
      A zero filled fast_weights of shape [batch_size, state_size, state_size]
    """
    state_size = self.state_size

    zeros = array_ops.zeros(tf.stack([batch_size, state_size, state_size]), dtype=dtype)
    zeros.set_shape([None, state_size, state_size])

    return zeros

def _create_tfrecord_dataset(tmpdir):
    if not gfile.Exists(tmpdir):
        gfile.MakeDirs(tmpdir)

    data_sources = test_utils.create_tfrecord_files(tmpdir, num_files=1)

    keys_to_features = {
        'image/encoded': tf.FixedLenFeature(shape=(), dtype=dtypes.string, default_value=''),
        'image/format': tf.FixedLenFeature(shape=(), dtype=dtypes.string, default_value='jpeg'),
        'image/class/label': tf.FixedLenFeature(
            shape=[1], dtype=dtypes.int64,
            default_value=array_ops.zeros([1], dtype=dtypes.int64))
    }

    items_to_handlers = {
        'image': tfslim.tfexample_decoder.Image(),
        'label': tfslim.tfexample_decoder.Tensor('image/class/label'),
    }

    decoder = TFExampleDecoder(keys_to_features, items_to_handlers)

    return Dataset(
        data_sources=data_sources, reader=tf.TFRecordReader, decoder=decoder, num_samples=100)

def _create_local(name, shape, collections=None, validate_shape=True,
                  dtype=tf.float32):
    """Creates a new local variable.
    Args:
        name: The name of the new or existing variable.
        shape: Shape of the new or existing variable.
        collections: A list of collection names to which the Variable will be added.
        validate_shape: Whether to validate the shape of the variable.
        dtype: Data type of the variables.
    Returns:
        The created variable.
    """
    # Make sure local variables are added to tf.GraphKeys.LOCAL_VARIABLES
    collections = list(collections or [])
    collections += [ops.GraphKeys.LOCAL_VARIABLES]
    return variables.Variable(
            initial_value=array_ops.zeros(shape, dtype=dtype),
            name=name,
            trainable=False,
            collections=collections,
            validate_shape=validate_shape)

def _create_local(name, shape, collections=None, validate_shape=True,
                  dtype=dtypes.float32):
    """Creates a new local variable.
    Args:
        name: The name of the new or existing variable.
        shape: Shape of the new or existing variable.
        collections: A list of collection names to which the Variable will be added.
        validate_shape: Whether to validate the shape of the variable.
        dtype: Data type of the variables.
    Returns:
        The created variable.
    """
    # Make sure local variables are added to tf.GraphKeys.LOCAL_VARIABLES
    collections = list(collections or [])
    collections += [ops.GraphKeys.LOCAL_VARIABLES]
    return variables.Variable(
            initial_value=array_ops.zeros(shape, dtype=dtype),
            name=name,
            trainable=False,
            collections=collections,
            validate_shape=validate_shape)

def testConstructionErrors(self):
    mu = [0., 0.]
    sigma = [1., 1.]
    self.assertRaises(
        ValueError,
        st.ObservedStochasticTensor,
        distributions.Normal(
            loc=mu, scale=sigma),
        value=array_ops.zeros((3,)))
    self.assertRaises(
        ValueError,
        st.ObservedStochasticTensor,
        distributions.Normal(
            loc=mu, scale=sigma),
        value=array_ops.zeros((3, 1)))
    self.assertRaises(
        ValueError,
        st.ObservedStochasticTensor,
        distributions.Normal(
            loc=mu, scale=sigma),
        value=array_ops.zeros(
            (1, 2), dtype=dtypes.int32))

def testStochasticVariablesWithCallablePriorInitializer(self):

    def prior_init(shape, dtype):
      return dist.Normal(
          array_ops.zeros(shape, dtype), array_ops.ones(shape, dtype))

    with variable_scope.variable_scope(
        "stochastic_variables",
        custom_getter=sv.make_stochastic_variable_getter(
            dist_cls=dist.NormalWithSoftplusScale, prior=prior_init)):
      w = variable_scope.get_variable("weights", (10, 20))

    x = random_ops.random_uniform((8, 10))
    y = math_ops.matmul(x, w)

    prior_map = vi._find_variational_and_priors(y, None)
    self.assertTrue(isinstance(prior_map[w], dist.Normal))
    elbo = vi.elbo(y, keep_batch_dim=False)

    with self.test_session() as sess:
      sess.run(variables.global_variables_initializer())
      sess.run(elbo)

def setUp(self):
    self._predictions = np.array([[4, 8, 12], [8, 1, 3]])
    self._labels = np.array([[1, 9, 2], [-5, -5, 7]])

    batch_size, dims = self._labels.shape

    # Compute the expected loss 'manually'.
    total = np.zeros((batch_size, 1))
    for b in range(batch_size):
      for i in range(dims):
        for j in range(dims):
          x = self._predictions[b, i].item() - self._predictions[b, j].item()
          y = self._labels[b, i].item() - self._labels[b, j].item()
          tmp = (x - y) * (x - y)
          total[b] += tmp

    self._expected_losses = np.divide(total, 9.0)

def testOutOfRangeDenseFeatures(self):
    with self._single_threaded_test_session():
      examples, variables = make_dense_examples_and_variables_dicts(
          dense_features_values=[[[1.0, 0.0], [0.0, 1.0]]],
          weights=[20.0, 10.0],
          labels=[1.0, 0.0])
      # Replace with a variable of size 1 instead of 2.
      variables['dense_features_weights'] = [
          variables_lib.Variable(array_ops.zeros(
              [1], dtype=dtypes.float32))
      ]
      options = dict(
          symmetric_l2_regularization=1.0,
          symmetric_l1_regularization=0,
          loss_type='logistic_loss')
      lr = SdcaModel(examples, variables, options)
      variables_lib.global_variables_initializer().run()
      train_op = lr.minimize()
      with self.assertRaisesRegexp(
          errors_impl.InvalidArgumentError,
          'More dense features than we have parameters for.*'):
        train_op.run()

  # TODO(katsiaspis): add a test for the case when examples at the end of an
  # epoch are repeated, since example id may be duplicated.

def testZeroLabelsPredictions(self):
    with self.test_session() as sess:
      predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
      labels = array_ops.zeros([4])
      thresholds = [0.5]
      prec, prec_op = metrics.streaming_precision_at_thresholds(predictions,
                                                                labels,
                                                                thresholds)
      rec, rec_op = metrics.streaming_recall_at_thresholds(predictions, labels,
                                                           thresholds)

      sess.run(variables.local_variables_initializer())
      sess.run([prec_op, rec_op])

      self.assertAlmostEqual(0, prec.eval(), 6)
      self.assertAlmostEqual(0, rec.eval(), 6)

def _shape_tensor(self):
    # Avoid messy broadcasting if possible.
    if self.shape.is_fully_defined():
      return ops.convert_to_tensor(
          self.shape.as_list(), dtype=dtypes.int32, name="shape")

    # Don't check the matrix dimensions.  That would add unnecessary Asserts to
    # the graph.  Things will fail at runtime naturally if shapes are
    # incompatible.
    matrix_shape = array_ops.stack([
        self.operators[0].range_dimension_tensor(),
        self.operators[-1].domain_dimension_tensor()
    ])

    # Dummy Tensor of zeros.  Will never be materialized.
    zeros = array_ops.zeros(shape=self.operators[0].batch_shape_tensor())
    for operator in self.operators[1:]:
      zeros += array_ops.zeros(shape=operator.batch_shape_tensor())
    batch_shape = array_ops.shape(zeros)

    return array_ops.concat((batch_shape, matrix_shape), 0)

def test_broadcast_apply_static_shapes(self):
    # These cannot be done in the automated (base test class) tests since they
    # test shapes that tf.batch_matmul cannot handle.
    # In particular, tf.batch_matmul does not broadcast.
    with self.test_session() as sess:
      # Given this x and LinearOperatorIdentity shape of (2, 1, 3, 3), the
      # broadcast shape of operator and 'x' is (2, 2, 3, 4)
      x = random_ops.random_normal(shape=(1, 2, 3, 4))
      operator = linalg_lib.LinearOperatorIdentity(
          num_rows=3, batch_shape=(2, 1), dtype=x.dtype)

      # Batch matrix of zeros with the broadcast shape of x and operator.
      zeros = array_ops.zeros(shape=(2, 2, 3, 4), dtype=x.dtype)

      # Expected result of apply and solve.
      expected = x + zeros

      operator_apply = operator.apply(x)
      self.assertAllEqual(operator_apply.get_shape(), expected.get_shape())
      self.assertAllClose(*sess.run([operator_apply, expected]))

def test_broadcast_apply_dynamic_shapes(self):
    # These cannot be done in the automated (base test class) tests since they
    # test shapes that tf.batch_matmul cannot handle.
    # In particular, tf.batch_matmul does not broadcast.
    with self.test_session() as sess:
      # Given this x and LinearOperatorIdentity shape of (2, 1, 3, 3), the
      # broadcast shape of operator and 'x' is (2, 2, 3, 4)
      x = array_ops.placeholder(dtypes.float32)
      num_rows = array_ops.placeholder(dtypes.int32)
      batch_shape = array_ops.placeholder(dtypes.int32)

      operator = linalg_lib.LinearOperatorIdentity(
          num_rows, batch_shape=batch_shape)
      feed_dict = {x: rng.rand(1, 2, 3, 4), num_rows: 3, batch_shape: (2, 1)}

      # Batch matrix of zeros with the broadcast shape of x and operator.
      zeros = array_ops.zeros(shape=(2, 2, 3, 4), dtype=x.dtype)

      # Expected result of apply and solve.
      expected = x + zeros

      operator_apply = operator.apply(x)
      self.assertAllClose(
          *sess.run([operator_apply, expected], feed_dict=feed_dict))

def testSamplingFromBatchOfNormals(self):
    batch_shape = (2,)
    with self.test_session():
      normal = distributions.Normal(
          loc=array_ops.zeros(
              batch_shape, dtype=dtypes.float32),
          scale=array_ops.ones(
              batch_shape, dtype=dtypes.float32))

      qdist = distributions.QuantizedDistribution(
          distribution=normal, lower_cutoff=0., upper_cutoff=None)

      samps = qdist.sample(5000, seed=42)
      samps_v = samps.eval()

      # With lower_cutoff = 0, the interval j=0 is (-infty, 0], which holds 1/2
      # of the mass of the normals.
      # rtol chosen to be 2x as large as necessary to pass.
      self.assertAllClose([0.5, 0.5], (samps_v == 0).mean(axis=0), rtol=0.03)

      # The interval j=1 is (0, 1], which is from the mean to one standard
      # deviation out.  This should contain 0.6827 / 2 of the mass.
      self.assertAllClose(
          [0.6827 / 2, 0.6827 / 2], (samps_v == 1).mean(axis=0), rtol=0.03)

def testDtypeAndShapeInheritedFromBaseDist(self):
    batch_shape = (2, 3)
    with self.test_session():
      qdist = distributions.QuantizedDistribution(
          distribution=distributions.Normal(
              loc=array_ops.zeros(batch_shape),
              scale=array_ops.zeros(batch_shape)),
          lower_cutoff=1.0,
          upper_cutoff=10.0)

      self.assertEqual(batch_shape, qdist.get_batch_shape())
      self.assertAllEqual(batch_shape, qdist.batch_shape().eval())
      self.assertEqual((), qdist.get_event_shape())
      self.assertAllEqual((), qdist.event_shape().eval())

      samps = qdist.sample(10, seed=42)
      self.assertEqual((10,) + batch_shape, samps.get_shape())
      self.assertAllEqual((10,) + batch_shape, samps.eval().shape)

      y = rng.randint(0, 5, size=batch_shape).astype(np.float32)
      self.assertEqual(batch_shape, qdist.prob(y).get_shape())

def testGrid2BasicLSTMCellWithRelu(self):
    with self.test_session() as sess:
      with variable_scope.variable_scope(
          'root', initializer=init_ops.constant_initializer(0.2)):
        x = array_ops.zeros([1, 3])
        m = array_ops.zeros([1, 4])
        cell = grid_rnn_cell.Grid2BasicLSTMCell(
            2, tied=False, non_recurrent_fn=nn_ops.relu)
        self.assertEqual(cell.state_size, 4)

        g, s = cell(x, m)
        self.assertEqual(g.get_shape(), (1, 2))
        self.assertEqual(s.get_shape(), (1, 4))

        sess.run([variables.global_variables_initializer()])
        res = sess.run(
            [g, s],
            {x: np.array([[1., 1., 1.]]),
             m: np.array([[0.1, 0.2, 0.3, 0.4]])})
        self.assertEqual(res[0].shape, (1, 2))
        self.assertEqual(res[1].shape, (1, 4))
        self.assertAllClose(res[0], [[0.31667367, 0.31667367]])
        self.assertAllClose(res[1], [[0.29530135, 0.37520045, 0.17044567,
                                      0.21292259]])

def testGrid2LSTMCell(self):
    with self.test_session() as sess:
      with variable_scope.variable_scope(
          'root', initializer=init_ops.constant_initializer(0.5)):
        x = array_ops.zeros([1, 3])
        m = array_ops.zeros([1, 8])
        cell = grid_rnn_cell.Grid2LSTMCell(2, use_peepholes=True)
        self.assertEqual(cell.state_size, 8)

        g, s = cell(x, m)
        self.assertEqual(g.get_shape(), (1, 2))
        self.assertEqual(s.get_shape(), (1, 8))

        sess.run([variables.global_variables_initializer()])
        res = sess.run([g, s], {
            x: np.array([[1., 1., 1.]]),
            m: np.array([[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]])
        })
        self.assertEqual(res[0].shape, (1, 2))
        self.assertEqual(res[1].shape, (1, 8))
        self.assertAllClose(res[0], [[0.95686918, 0.95686918]])
        self.assertAllClose(res[1], [[2.41515064, 2.41515064, 0.95686918,
                                      0.95686918, 1.38917875, 1.49043763,
                                      0.83884692, 0.86036491]])

def testGrid2LSTMCellTied(self):
    with self.test_session() as sess:
      with variable_scope.variable_scope(
          'root', initializer=init_ops.constant_initializer(0.5)):
        x = array_ops.zeros([1, 3])
        m = array_ops.zeros([1, 8])
        cell = grid_rnn_cell.Grid2LSTMCell(2, tied=True, use_peepholes=True)
        self.assertEqual(cell.state_size, 8)

        g, s = cell(x, m)
        self.assertEqual(g.get_shape(), (1, 2))
        self.assertEqual(s.get_shape(), (1, 8))

        sess.run([variables.global_variables_initializer()])
        res = sess.run([g, s], {
            x: np.array([[1., 1., 1.]]),
            m: np.array([[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]])
        })
        self.assertEqual(res[0].shape, (1, 2))
        self.assertEqual(res[1].shape, (1, 8))
        self.assertAllClose(res[0], [[0.95686918, 0.95686918]])
        self.assertAllClose(res[1], [[2.41515064, 2.41515064, 0.95686918,
                                      0.95686918, 1.38917875, 1.49043763,
                                      0.83884692, 0.86036491]])

def testGrid2LSTMCellWithRelu(self):
    with self.test_session() as sess:
      with variable_scope.variable_scope(
          'root', initializer=init_ops.constant_initializer(0.5)):
        x = array_ops.zeros([1, 3])
        m = array_ops.zeros([1, 4])
        cell = grid_rnn_cell.Grid2LSTMCell(
            2, use_peepholes=True, non_recurrent_fn=nn_ops.relu)
        self.assertEqual(cell.state_size, 4)

        g, s = cell(x, m)
        self.assertEqual(g.get_shape(), (1, 2))
        self.assertEqual(s.get_shape(), (1, 4))

        sess.run([variables.global_variables_initializer()])
        res = sess.run(
            [g, s],
            {x: np.array([[1., 1., 1.]]),
             m: np.array([[0.1, 0.2, 0.3, 0.4]])})
        self.assertEqual(res[0].shape, (1, 2))
        self.assertEqual(res[1].shape, (1, 4))
        self.assertAllClose(res[0], [[2.1831727, 2.1831727]])
        self.assertAllClose(res[1], [[0.92270052, 1.02325559, 0.66159075,
                                      0.70475441]])

def testGrid2BasicRNNCellTied(self):
    with self.test_session() as sess:
      with variable_scope.variable_scope(
          'root', initializer=init_ops.constant_initializer(0.5)):
        x = array_ops.zeros([2, 2])
        m = array_ops.zeros([2, 4])
        cell = grid_rnn_cell.Grid2BasicRNNCell(2, tied=True)
        self.assertEqual(cell.state_size, 4)

        g, s = cell(x, m)
        self.assertEqual(g.get_shape(), (2, 2))
        self.assertEqual(s.get_shape(), (2, 4))

        sess.run([variables.global_variables_initializer()])
        res = sess.run([g, s], {
            x: np.array([[1., 1.], [2., 2.]]),
            m: np.array([[0.1, 0.1, 0.1, 0.1], [0.2, 0.2, 0.2, 0.2]])
        })
        self.assertEqual(res[0].shape, (2, 2))
        self.assertEqual(res[1].shape, (2, 4))
        self.assertAllClose(res[0], [[0.94685763, 0.94685763],
                                     [0.99480951, 0.99480951]])
        self.assertAllClose(res[1],
                            [[0.94685763, 0.94685763, 0.80049908, 0.80049908],
                             [0.99480951, 0.99480951, 0.97574311, 0.97574311]])

def testGrid2BasicRNNCellWithRelu(self):
    with self.test_session() as sess:
      with variable_scope.variable_scope(
          'root', initializer=init_ops.constant_initializer(0.5)):
        x = array_ops.zeros([1, 2])
        m = array_ops.zeros([1, 2])
        cell = grid_rnn_cell.Grid2BasicRNNCell(2, non_recurrent_fn=nn_ops.relu)
        self.assertEqual(cell.state_size, 2)

        g, s = cell(x, m)
        self.assertEqual(g.get_shape(), (1, 2))
        self.assertEqual(s.get_shape(), (1, 2))

        sess.run([variables.global_variables_initializer()])
        res = sess.run([g, s],
                       {x: np.array([[1., 1.]]),
                        m: np.array([[0.1, 0.1]])})
        self.assertEqual(res[0].shape, (1, 2))
        self.assertEqual(res[1].shape, (1, 2))
        self.assertAllClose(res[0], [[1.80049896, 1.80049896]])
        self.assertAllClose(res[1], [[0.80049896, 0.80049896]])

def testGridRNNEdgeCasesLikeRelu(self):
    with self.test_session() as sess:
      with variable_scope.variable_scope(
          'root', initializer=init_ops.constant_initializer(0.5)):
        x = array_ops.zeros([3, 2])
        m = array_ops.zeros([0, 0])

        # this is equivalent to relu
        cell = grid_rnn_cell.GridRNNCell(
            num_units=2,
            num_dims=1,
            input_dims=0,
            output_dims=0,
            non_recurrent_dims=0,
            non_recurrent_fn=nn_ops.relu)
        g, s = cell(x, m)
        self.assertEqual(g.get_shape(), (3, 2))
        self.assertEqual(s.get_shape(), (0, 0))

        sess.run([variables.global_variables_initializer()])
        res = sess.run([g, s], {x: np.array([[1., -1.], [-2, 1], [2, -1]])})
        self.assertEqual(res[0].shape, (3, 2))
        self.assertEqual(res[1].shape, (0, 0))
        self.assertAllClose(res[0], [[0, 0], [0, 0], [0.5, 0.5]])

def benchmarkTfRNNLSTMBlockCellTraining(self):
    test_configs = self._GetTestConfig()
    for config_name, config in test_configs.items():
      num_layers = config["num_layers"]
      num_units = config["num_units"]
      batch_size = config["batch_size"]
      seq_length = config["seq_length"]

      with ops.Graph().as_default(), ops.device("/gpu:0"):
        inputs = seq_length * [
            array_ops.zeros([batch_size, num_units], dtypes.float32)
        ]
        cell = lambda: lstm_ops.LSTMBlockCell(num_units=num_units)  # pylint: disable=cell-var-from-loop

        multi_cell = core_rnn_cell_impl.MultiRNNCell(
            [cell() for _ in range(num_layers)])
        outputs, final_state = core_rnn.static_rnn(
            multi_cell, inputs, dtype=dtypes.float32)
        trainable_variables = ops.get_collection(
            ops.GraphKeys.TRAINABLE_VARIABLES)
        gradients = gradients_impl.gradients([outputs, final_state],
                                             trainable_variables)
        training_op = control_flow_ops.group(*gradients)
        self._BenchmarkOp(training_op, "tf_rnn_lstm_block_cell %s %s" %
                          (config_name, self._GetConfigDesc(config)))

def _testSaveRestoreVariable(self, rnn_mode):
    model = self._CreateModel(rnn_mode, num_layers=2, num_units=7, input_size=3)
    random_seed.set_random_seed(1234)
    params_size_t = model.params_size()
    params = variables.Variable(
        random_ops.random_uniform([params_size_t]), validate_shape=False)
    self._create_params_savable(params, model)
    save_path = os.path.join(self.get_temp_dir(), "save-restore-variable-test")
    saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
    with self.test_session(use_gpu=True) as sess:
      sess.run(variables.global_variables_initializer())
      params_v = sess.run(params)
      val = saver.save(sess, save_path)
      self.assertEqual(save_path, val)
    with self.test_session(use_gpu=True) as sess:
      reset_params = state_ops.assign(params, array_ops.zeros([params_size_t]))
      sess.run(reset_params)
      saver.restore(sess, save_path)
      params_v_restored = sess.run(params)
      self.assertAllEqual(params_v, params_v_restored)

def testLSTMLayerErrors(self):
    num_inputs = 1
    num_nodes = 1
    seq_length = 3

    weights = array_ops.zeros(lstm.LSTMCellWeightsShape(num_inputs, num_nodes))
    m = constant_op.constant([[0.]] * self._batch_size)
    c = constant_op.constant([[0.]] * self._batch_size)
    x_seq = [constant_op.constant(self._inputs)] * seq_length
    pad = constant_op.constant([[0.]] * self._batch_size)

    with self.assertRaisesWithPredicateMatch(ValueError, 'length of x_seq'):
      lstm.LSTMLayer('lstm', weights, m, c, x_seq, [pad])
    with self.assertRaisesWithPredicateMatch(ValueError, 'length of x_seq'):
      lstm.LSTMLayer('lstm', weights, m, c, x_seq, [pad] * 2)
    with self.assertRaisesWithPredicateMatch(ValueError, 'length of x_seq'):
      lstm.LSTMLayer('lstm', weights, m, c, x_seq, [pad] * 4)

def BuildLSTMLayer(batch_size, seq_length, num_inputs, num_nodes):
  """Builds a single LSTM layer with random weights and inputs.

  Args:
    batch_size: Inputs are fed in batches of this size.
    seq_length: The sequence length to unroll the LSTM layer.
    num_inputs: Dimension of inputs that are fed into each LSTM cell.
    num_nodes: The number of nodes in each LSTM cell.

  Returns:
    (out_seq, weights) pair.  The out_seq is a list of per-sequence-step
    outputs, each with shape [batch_size, num_nodes].  The weights are a list of
    weight variables that may be trained.
  """
  weights = RandomVar(
      LSTMCellWeightsShape(num_inputs, num_nodes), name='weights')
  m = array_ops.zeros([batch_size, num_nodes], name='init_m')
  c = array_ops.zeros([batch_size, num_nodes], name='init_c')
  x_seq, pad_seq = RandomInputs(batch_size, seq_length, num_inputs)

  out_seq = LSTMLayer('lstm', weights, m, c, x_seq, pad_seq)
  return out_seq, [weights]

def _create_zero_outputs(size, dtype, batch_size):
  """Create a zero outputs Tensor structure."""
  def _t(s):
    return (s if isinstance(s, ops.Tensor) else constant_op.constant(
        tensor_shape.TensorShape(s).as_list(),
        dtype=dtypes.int32,
        name="zero_suffix_shape"))

  def _create(s, d):
    return array_ops.zeros(
        array_ops.concat(
            ([batch_size], _t(s)), axis=0), dtype=d)

  return nest.map_structure(_create, size, dtype)

def zero_state(self, batch_size, dtype):
    with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]):
      if self._initial_cell_state is not None:
        cell_state = self._initial_cell_state
      else:
        cell_state = self._cell.zero_state(batch_size, dtype)
      error_message = (
          "When calling zero_state of AttentionWrapper %s: " % self._base_name +
          "Non-matching batch sizes between the memory "
          "(encoder output) and the requested batch size.  Are you using "
          "the BeamSearchDecoder?  If so, make sure your encoder output has "
          "been tiled to beam_width via tf.contrib.seq2seq.tile_batch, and "
          "the batch_size= argument passed to zero_state is "
          "batch_size * beam_width.")
      with ops.control_dependencies(
          [check_ops.assert_equal(batch_size,
                                  self._attention_mechanism.batch_size,
                                  message=error_message)]):
        cell_state = nest.map_structure(
            lambda s: array_ops.identity(s, name="checked_cell_state"),
            cell_state)
      if self._alignment_history:
        alignment_history = tensor_array_ops.TensorArray(
            dtype=dtype, size=0, dynamic_size=True)
      else:
        alignment_history = ()
      return AttentionWrapperState(
          cell_state=cell_state,
          time=array_ops.zeros([], dtype=dtypes.int32),
          attention=_zero_state_tensors(self._attention_size, batch_size,
                                        dtype),
          alignments=self._attention_mechanism.initial_alignments(
              batch_size, dtype),
          alignment_history=alignment_history)

def _create_zero_outputs(size, dtype, batch_size):
  """Create a zero outputs Tensor structure."""
  def _t(s):
    return (s if isinstance(s, ops.Tensor) else constant_op.constant(
        tensor_shape.TensorShape(s).as_list(),
        dtype=dtypes.int32,
        name="zero_suffix_shape"))

  def _create(s, d):
    return array_ops.zeros(
        array_ops.concat(
            ([batch_size], _t(s)), axis=0), dtype=d)

  return nest.map_structure(_create, size, dtype)

def _MaxPoolGradGrad(op, grad):
    gradient = gen_nn_ops._max_pool_grad(op.inputs[0], op.outputs[0],
            grad, op.get_attr("ksize"), op.get_attr("strides"),
            padding=op.get_attr("padding"), data_format=op.get_attr("data_format"))
    gradgrad1 = array_ops.zeros(shape = array_ops.shape(op.inputs[1]), dtype=gradient.dtype)
    gradgrad2 = array_ops.zeros(shape = array_ops.shape(op.inputs[2]), dtype=gradient.dtype)
    return (gradient, gradgrad1, gradgrad2)

def _create_zero_outputs(size, dtype, batch_size):
    """Create a zero outputs Tensor structure."""
    def _t(s):
        return (s if isinstance(s, ops.Tensor) else constant_op.constant(
            tensor_shape.TensorShape(s).as_list(),
            dtype=dtypes.int32,
            name="zero_suffix_shape"))

    def _create(s, d):
        return array_ops.zeros(
            array_ops.concat(
                ([batch_size], _t(s)), axis=0), dtype=d)

    return nest.map_structure(_create, size, dtype)

def zeros(shape, dtype=None, name=None):
      """Instantiates an all-zeros variable and returns it.

      Arguments:
          shape: Tuple of integers, shape of returned Keras variable
          dtype: String, data type of returned Keras variable
          name: String, name of returned Keras variable

      Returns:
          A variable (including Keras metadata), filled with `0.0`.

      Example:
      ```python
          >>> from keras import backend as K
          >>> kvar = K.zeros((3,4))
          >>> K.eval(kvar)
          array([[ 0.,  0.,  0.,  0.],
                 [ 0.,  0.,  0.,  0.],
                 [ 0.,  0.,  0.,  0.]], dtype=float32)

def temporal_padding(x, padding=(1, 1)):
      """Pads the middle dimension of a 3D tensor.

      Arguments:
          x: Tensor or variable.
          padding: Tuple of 2 integers, how many zeros to
              add at the start and end of dim 1.

      Returns:
          A padded 3D tensor.
      """
      assert len(padding) == 2
      pattern = [[0, 0], [padding[0], padding[1]], [0, 0]]
      return array_ops.pad(x, pattern)

def spatial_3d_padding(x, padding=((1, 1), (1, 1), (1, 1)), data_format=None):
      """Pads 5D tensor with zeros along the depth, height, width dimensions.

      Pads these dimensions with respectively
      "padding[0]", "padding[1]" and "padding[2]" zeros left and right.

      For 'channels_last' data_format,
      the 2nd, 3rd and 4th dimension will be padded.
      For 'channels_first' data_format,
      the 3rd, 4th and 5th dimension will be padded.

      Arguments:
          x: Tensor or variable.
          padding: Tuple of 3 tuples, padding pattern.
          data_format: One of `channels_last` or `channels_first`.

      Returns:
          A padded 5D tensor.

      Raises:
          ValueError: if `data_format` is neither
              `channels_last` or `channels_first`.

      """
      assert len(padding) == 3
      assert len(padding[0]) == 2
      assert len(padding[1]) == 2
      assert len(padding[2]) == 2
      if data_format is None:
        data_format = image_data_format()
      if data_format not in {'channels_first', 'channels_last'}:
        raise ValueError('Unknown data_format ' + str(data_format))

      if data_format == 'channels_first':
        pattern = [[0, 0], [0, 0], [padding[0][0], padding[0][1]],
                   [padding[1][0], padding[1][1]], [padding[2][0], padding[2][1]]]
      else:
        pattern = [[0, 0], [padding[0][0], padding[0][1]],
                   [padding[1][0], padding[1][1]], [padding[2][0],
                                                    padding[2][1]], [0, 0]]
      return array_ops.pad(x, pattern)

def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        self.updates = []

        lr = self.lr
        if self.initial_decay > 0:
          lr *= (1. / (1. + self.decay * self.iterations))
          self.updates.append(K.update_add(self.iterations, 1))

        # momentum
        shapes = [K.int_shape(p) for p in params]
        moments = [K.zeros(shape) for shape in shapes]
        self.weights = [self.iterations] + moments
        for p, g, m in zip(params, grads, moments):
          v = self.momentum * m - lr * g  # velocity
          self.updates.append(K.update(m, v))

          if self.nesterov:
            new_p = p + self.momentum * v - lr * g
          else:
            new_p = p + v

          # apply constraints
          if p in constraints:
            c = constraints[p]
            new_p = c(new_p)

          self.updates.append(K.update(p, new_p))
        return self.updates

def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        shapes = [K.int_shape(p) for p in params]
        accumulators = [K.zeros(shape) for shape in shapes]
        delta_accumulators = [K.zeros(shape) for shape in shapes]
        self.weights = accumulators + delta_accumulators
        self.updates = []

        lr = self.lr
        if self.initial_decay > 0:
          lr *= (1. / (1. + self.decay * self.iterations))
          self.updates.append(K.update_add(self.iterations, 1))

        for p, g, a, d_a in zip(params, grads, accumulators, delta_accumulators):
          # update accumulator
          new_a = self.rho * a + (1. - self.rho) * K.square(g)
          self.updates.append(K.update(a, new_a))

          # use the new accumulator and the *old* delta_accumulator
          update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)

          new_p = p - lr * update
          # apply constraints
          if p in constraints:
            c = constraints[p]
            new_p = c(new_p)
          self.updates.append(K.update(p, new_p))

          # update delta_accumulator
          new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update)
          self.updates.append(K.update(d_a, new_d_a))
        return self.updates

def get_updates(self, params, constraints, loss):
        grads = self.get_gradients(loss, params)
        self.updates = [K.update_add(self.iterations, 1)]

        lr = self.lr
        if self.initial_decay > 0:
          lr *= (1. / (1. + self.decay * self.iterations))

        t = self.iterations + 1
        lr_t = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
                     (1. - K.pow(self.beta_1, t)))

        shapes = [K.int_shape(p) for p in params]
        ms = [K.zeros(shape) for shape in shapes]
        vs = [K.zeros(shape) for shape in shapes]
        self.weights = [self.iterations] + ms + vs

        for p, g, m, v in zip(params, grads, ms, vs):
          m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
          v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
          p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)

          self.updates.append(K.update(m, m_t))
          self.updates.append(K.update(v, v_t))

          new_p = p_t
          # apply constraints
          if p in constraints:
            c = constraints[p]
            new_p = c(new_p)
          self.updates.append(K.update(p, new_p))
        return self.updates