Python numpy 模块，array() 实例源码
我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用numpy.array()。
def min_side(_, pos):
"""
Given an object pixels' positions, return the minimum side length of its
bounding box
:param _: pixel values (unused)
:param pos: pixel position (1-D)
:return: minimum bounding box side length
"""
xs = np.array([i / SSIZE for i in pos])
ys = np.array([i % SSIZE for i in pos])
minx = np.amin(xs)
miny = np.amin(ys)
maxx = np.amax(xs)
maxy = np.amax(ys)
ct1 = compute_line(np.array([minx, miny]), np.array([minx, maxy]))
ct2 = compute_line(np.array([minx, miny]), np.array([maxx, miny]))
return min(ct1, ct2)
def _cascade_evaluation(self, X_test, y_test):
""" Evaluate the accuracy of the cascade using X and y.
:param X_test: np.array
Array containing the test input samples.
Must be of the same shape as training data.
:param y_test: np.array
Test target values.
:return: float
the cascade accuracy.
"""
casc_pred_prob = np.mean(self.cascade_forest(X_test), axis=0)
casc_pred = np.argmax(casc_pred_prob, axis=1)
casc_accuracy = accuracy_score(y_true=y_test, y_pred=casc_pred)
print('Layer validation accuracy = {}'.format(casc_accuracy))
return casc_accuracy
def _create_feat_arr(self, X, prf_crf_pred):
""" Concatenate the original feature vector with the predicition probabilities
of a cascade layer.
:param X: np.array
Array containing the input samples.
Must be of shape [n_samples, data] where data is a 1D array.
:param prf_crf_pred: list
Prediction probabilities by a cascade layer for X.
:return: np.array
Concatenation of X and the predicted probabilities.
To be used for the next layer in a cascade forest.
"""
swap_pred = np.swapaxes(prf_crf_pred, 0, 1)
add_feat = swap_pred.reshape([np.shape(X)[0], -1])
feat_arr = np.concatenate([add_feat, X], axis=1)
return feat_arr
def shuffleBlock(self,cells,d,tlx,tly,cols,rows,width,height):
if tlx+cols < width and tly+rows < height:
temp = []
for row in range( rows):
for col in range( cols):
temp.append(d[cells[tlx+col][tly+row]])
temp = np.array(temp)
oldState = temp.copy()
np.random.shuffle(temp)
i = 0
for row in range( rows):
for col in range( cols):
d[cells[tlx+col][tly+row]] = temp[i]
i+=1
return oldState
else:
return []
def train(self, dataset, train_split=0.8, dense_size=32, learning_rate=0.001, batch_size=32, epochs=50, activation='relu'):
self.__load_dataset(dataset, train_split)
train_x = np.array(self.__train_data[:, 0].tolist())
train_y = to_categorical(self.__train_data[:, 1], 2)
test_x = np.array(self.__test_data[:, 0].tolist())
test_y = to_categorical(self.__test_data[:, 1], 2)
print(train_x.shape)
self.__model = Sequential()
self.__model.add(Dense(dense_size, input_dim=train_x.shape[1], activation=activation, init='glorot_uniform'))
self.__model.add(Dense(train_y.shape[1], activation='softmax', init='glorot_uniform'))
self.__model.compile(optimizer=Adam(lr=0.001), loss='categorical_crossentropy', metrics=['categorical_accuracy'])
self.__model.fit(train_x, train_y, batch_size=batch_size, nb_epoch=epochs, validation_data=(test_x, test_y), verbose=2)
def normalize_array (solution, prediction):
''' Use min and max of solution as scaling factors to normalize prediction,
then threshold it to [0, 1]. Binarize solution to {0, 1}.
This allows applying classification scores to all cases.
In principle, this should not do anything to properly formatted
classification inputs and outputs.'''
# Binarize solution
sol=np.ravel(solution) # convert to 1-d array
maxi = np.nanmax((filter(lambda x: x != float('inf'), sol))) # Max except NaN and Inf
mini = np.nanmin((filter(lambda x: x != float('-inf'), sol))) # Mini except NaN and Inf
if maxi == mini:
print('Warning, cannot normalize')
return [solution, prediction]
diff = maxi - mini
mid = (maxi + mini)/2.
new_solution = np.copy(solution)
new_solution[solution>=mid] = 1
new_solution[solution<mid] = 0
# Normalize and threshold predictions (takes effect only if solution not in {0, 1})
new_prediction = (np.copy(prediction) - float(mini))/float(diff)
new_prediction[new_prediction>1] = 1 # and if predictions exceed the bounds [0, 1]
new_prediction[new_prediction<0] = 0
# Make probabilities smoother
#new_prediction = np.power(new_prediction, (1./10))
return [new_solution, new_prediction]
def mvmean(R, axis=0):
''' Moving average to avoid rounding errors. A bit slow, but...
Computes the mean along the given axis, except if this is a vector, in which case the mean is returned.
Does NOT flatten.'''
if len(R.shape)==0: return R
average = lambda x: reduce(lambda i, j: (0, (j[0]/(j[0]+1.))*i[1]+(1./(j[0]+1))*j[1]), enumerate(x))[1]
R=np.array(R)
if len(R.shape)==1: return average(R)
if axis==1:
return np.array(map(average, R))
else:
return np.array(map(average, R.transpose()))
# ======= All metrics used for scoring in the challenge ========
### REGRESSION METRICS (work on raw solution and prediction)
# These can be computed on all solutions and predictions (classification included)
def data_binary_sparse (filename, nbr_features):
''' This function takes as an argument a file representing a binary sparse matrix
binary_sparse_matrix[i][j] = a means matrix[i][j] = 1
It converts it into a numpy array an returns this array. '''
data = data_converter.file_to_array (filename)
nbr_samples = len(data)
dok_sparse = dok_matrix ((nbr_samples, nbr_features)) # the construction is easier w/ dok_sparse
print ("Converting {} to dok sparse matrix".format(filename))
for row in range (nbr_samples):
for feature in data[row]:
dok_sparse[row, int(feature)-1] = 1
print ("Converting {} to csr sparse matrix".format(filename))
return dok_sparse.tocsr()
# ================ Copy results from input to output ==========================
def generate_one_summary(self, review):
"""
Create summary for one review using Encoder Decoder Seq2Seq model
:param review: The input review
:return: Output Summary of the model
"""
review = review.T
review = [np.array([int(x)]) for x in review]
feed_dict_rev = {self.enc_inp[t]: review[t] for t in range(self.seq_length)}
feed_dict_rev.update({self.labels[t]: review[t] for t in range(self.seq_length)})
summary = self.sess.run(self.dec_outputs_tst, feed_dict_rev)
summary = [logits_t.argmax(axis=1) for logits_t in summary]
summary = [x[0] for x in summary]
return summary
def generate_one_summary(self, review):
"""
Create summary for one review using Encoder Decoder Seq2Seq model
:param review: The input review
:return: Output Summary of the model
"""
review = review.T
review = [np.array([int(x)]) for x in review]
feed_dict_rev = {self.enc_inp[t]: review[t] for t in range(self.seq_length)}
feed_dict_rev.update({self.labels[t]: review[t] for t in range(self.seq_length)})
summary = self.sess.run(self.dec_outputs_tst, feed_dict_rev)
summary = [logits_t.argmax(axis=1) for logits_t in summary]
summary = [x[0] for x in summary]
return summary
def __crawl_review(self):
"""
Crawl review
:return: review [numpy array]
"""
review_list = []
print 'Crawling Reviews....'
num_lines = 0
with open(self.raw_data_file) as infile:
for line in infile:
if line.startswith('review/text'):
if num_lines >= self.num_reviews:
break
num_lines += 1
_,review = line.split('/text: ')
review_list.append(review)
return np.array(review_list)
def __crawl_summary(self):
"""
Crawl summary
:return: summary [numpy array]
"""
summary_list = []
print 'Crawling Summary....'
num_lines = 0
with open(self.raw_data_file) as infile:
for line in infile:
if line.startswith('review/summary'):
if num_lines >= self.num_reviews:
break
num_lines += 1
_,summary = line.split('/summary: ')
summary_list.append(summary)
return np.array(summary_list)
def reshape_array(array, newsize, pixcombine='sum'):
"""
Reshape an array to a give size using either the sum, mean or median of the pixels binned
Note that the old array dimensions have to be multiples of the new array dimensions
--- INPUT ---
array Array to reshape (combine pixels)
newsize New size of array
pixcombine The method to combine the pixels with. Choices are sum, mean and median
"""
sh = newsize[0],array.shape[0]//newsize[0],newsize[1],array.shape[1]//newsize[1]
pdb.set_trace()
if pixcombine == 'sum':
reshapedarray = array.reshape(sh).sum(-1).sum(1)
elif pixcombine == 'mean':
reshapedarray = array.reshape(sh).mean(-1).mean(1)
elif pixcombine == 'median':
reshapedarray = array.reshape(sh).median(-1).median(1)
return reshapedarray
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
def test(path_test, input_size, hidden_size, batch_size, save_dir, model_name, maxlen):
db = read_data(path_test)
X = create_sequences(db[:-maxlen], win_size=maxlen, step=maxlen)
X = np.reshape(X, (X.shape[0], X.shape[1], input_size))
# build the model: 1 layer LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(hidden_size, return_sequences=False, input_shape=(maxlen, input_size)))
model.add(Dense(maxlen))
model.load_weights(save_dir + model_name)
model.compile(loss='mse', optimizer='adam')
prediction = model.predict(X, batch_size, verbose=1)
prediction = prediction.flatten()
# prediction_container = np.array(prediction).flatten()
Y = db[maxlen:]
plt.plot(prediction, label='prediction')
plt.plot(Y, label='true')
plt.legend()
plt.show()
def word_list_to_embedding(words, embeddings, embedding_dimension=50):
'''
:param words: an n x (2*window_size + 1) matrix from data_to_mat
:param embeddings: an embedding dictionary where keys are strings and values
are embeddings; the output from embeddings_to_dict
:param embedding_dimension: the dimension of the values in embeddings; in this
assignment, embedding_dimension=50
:return: an n x ((2*window_size + 1)*embedding_dimension) matrix where each entry of the
words matrix is replaced with its embedding
'''
m, n = words.shape
words = words.reshape((-1))
return np.array([embeddings[w] for w in words], dtype=np.float32).reshape(m, n*embedding_dimension)
#
# End Twitter Helper Functions
#
def __init__(self, N, L, comm, precision,
communication="Alltoall",
padsize=1.5,
threads=1,
planner_effort=defaultdict(lambda: "FFTW_MEASURE")):
R2C.__init__(self, N, L, comm, precision,
communication=communication,
padsize=padsize, threads=threads,
planner_effort=planner_effort)
# Reuse all shapes from r2c transform R2C simply by resizing the final complex z-dimension:
self.Nf = N[2]
self.Nfp = int(self.padsize*self.N[2]) # Independent complex wavenumbers in z-direction for padded array
# Rename since there's no real space
self.original_shape_padded = self.real_shape_padded
self.original_shape = self.real_shape
self.transformed_shape = self.complex_shape
self.original_local_slice = self.real_local_slice
self.transformed_local_slice = self.complex_local_slice
self.ks = (fftfreq(N[2])*N[2]).astype(int)
def bag_of_tokens(config, labels, label_lengths):
if config.train_output_embeddings:
with tf.variable_scope('embed', reuse=True):
output_embeddings = tf.get_variable('output_embedding')
else:
output_embeddings = tf.constant(config.output_embedding_matrix)
#everything_label_placeholder = tf.placeholder(shape=(None, config.max_length,), dtype=tf.int32)
#everything_label_length_placeholder = tf.placeholder(shape=(None,), dtype=tf.int32)
labels = tf.constant(np.array(labels))
embedded_output = tf.gather(output_embeddings, labels)
print('embedded_output before', embedded_output)
#mask = tf.sequence_mask(label_lengths, maxlen=config.max_length, dtype=tf.float32)
# note: this multiplication will broadcast the mask along all elements of the depth dimension
# (which is why we run the expand_dims to choose how to broadcast)
#embedded_output = embedded_output * tf.expand_dims(mask, axis=2)
#print('embedded_output after', embedded_output)
return tf.reduce_sum(embedded_output, axis=1)
def _compute_process_and_covariance_matrices(self, dt):
"""Computes the transition and covariance matrix of the process model and measurement model.
Args:
dt (float): Timestep of the discrete transition.
Returns:
F (numpy.ndarray): Transition matrix.
Q (numpy.ndarray): Process covariance matrix.
R (numpy.ndarray): Measurement covariance matrix.
"""
F = np.array(np.bmat([[np.eye(3), dt * np.eye(3)], [np.zeros((3, 3)), np.eye(3)]]))
self.process_matrix = F
q_p = self.process_covariance_position
q_v = self.process_covariance_velocity
Q = np.diag([q_p, q_p, q_p, q_v, q_v, q_v]) ** 2 * dt
r = self.measurement_covariance
R = r * np.eye(4)
self.process_covariance = Q
self.measurement_covariance = R
return F, Q, R
def sample(self, sample_size=20, text=None):
"""Sample the documents."""
p = 1
if text != None:
try:
x, word_idxs = self.reader.get(text)
except Exception as e:
print(e)
return
else:
x, word_idxs = self.reader.random()
print(" [*] Text: %s" % " ".join([self.reader.idx2word[word_idx] for word_idx in word_idxs]))
cur_ps = self.sess.run(self.p_x_i, feed_dict={self.x: x})
word_idxs = np.array(cur_ps).argsort()[-sample_size:][::-1]
ps = cur_ps[word_idxs]
for idx, (cur_p, word_idx) in enumerate(zip(ps, word_idxs)):
print(" [%d] %-20s: %.8f" % (idx+1, self.reader.idx2word[word_idx], cur_p))
p *= cur_p
print(" [*] perp : %8.f" % -np.log(p))
def plot_nucleotide_diversity(ax, fqlists, invert=False):
'''
Create a FastQC-like "?Per base sequence content" plot
Plot fraction of nucleotides per position
zip will stop when shortest read is exhausted
'''
if invert:
fqlists = [list(reversed(read)) for read in fqlists]
numreads = len(fqlists)
sns.set_style("darkgrid")
l_A, = ax.plot(
np.array([pos.count('A') / numreads for pos in zip(*fqlists)]), 'green', label='A')
l_T, = ax.plot(
np.array([pos.count('T') / numreads for pos in zip(*fqlists)]), 'red', label='T')
l_G, = ax.plot(
np.array([pos.count('G') / numreads for pos in zip(*fqlists)]), 'black', label='G')
l_C, = ax.plot(
np.array([pos.count('C') / numreads for pos in zip(*fqlists)]), 'blue', label='C')
if invert:
ax.set_xticklabels(-1 * ax.get_xticks().astype(int))
return [l_A, l_T, l_G, l_C]
def plot_qual(ax, quallist, invert=False):
'''
Create a FastQC-like "?Per base sequence quality?" plot
Plot average quality per position
zip will stop when shortest read is exhausted
'''
sns.set_style("darkgrid")
if invert:
l_Q, = ax.plot(np.array([np.mean(position) for position in zip(
*[list(reversed(read)) for read in quallist])]), 'orange', label="Quality")
ax.set_xlabel('Position in read from end')
ax.set_xticklabels(-1 * ax.get_xticks().astype(int))
else:
l_Q, = ax.plot(np.array([np.mean(position)
for position in zip(*quallist)]), 'orange', label="Quality")
ax.set_xlabel('Position in read from start')
return l_Q
def d_x2(self, factors=None):
"""Creates a sparse matrix for computing the second derivative with respect to x multiplied
by factors given for every point. Uses central difference quotient.
Args:
factors: Factor for each point to be applied after derivation.
Returns:
Sparse matrix the calculate second derivatives of field components.
"""
# use ones as factors if none are specified
if factors is None:
factors = np.array(1).repeat(self.num_points)
return sp.dia_matrix((np.array([factors, -2*factors, factors]), [-1, 0, 1]),
shape=(self.num_points, self.num_points))
def d_x2(self, factors=None):
"""Creates a sparse matrix for computing the second derivative with respect to x multiplied
by factors given for every point. Uses central difference quotient.
Args:
factors: Factor for each point to be applied after derivation.
Returns:
Sparse matrix the calculate second derivatives of field components.
"""
# use ones as factors if none are specified
if factors is None:
factors = np.array(1).repeat(self.num_points)
return sp.dia_matrix((np.array([factors, -2*factors, factors]), [-1, 0, 1]),
shape=(self.num_points, self.num_points))
def plot_region(self, region):
"""Shows the given region in the field plot.
Args:
region: Region to be plotted.
"""
if type(region) == reg.PointRegion:
self.axes.plot(np.ones(2) * region.point_coordinates / self._x_axis_factor,
np.array([-1, 1]) * self.scale, color='black')
elif type(region) == reg.LineRegion:
self.axes.plot(np.ones(2) * region.line_coordinates[0] / self._x_axis_factor,
np.array([-1, 1]) * self.scale, color='black')
self.axes.plot(np.ones(2) * region.line_coordinates[1] / self._x_axis_factor,
np.array([-1, 1]) * self.scale, color='black')
else:
raise TypeError('Unknown type in region list: {}'.format(type(region)))
def test_accuracy_full_batch(tokens, features, mini_batch_size, word_attn, sent_attn, th=0.5):
p = []
l = []
cnt = 0
g = gen_minibatch1(tokens, features, mini_batch_size, False)
for token, feature in g:
if cnt % 100 == 0:
print(cnt)
cnt +=1
# print token.size()
# y_pred = get_predictions(token, word_attn, sent_attn)
# print y_pred
y_pred = get_predictions(token, feature, word_attn, sent_attn)
# print y_pred
# _, y_pred = torch.max(y_pred, 1)
# y_pred = y_pred[:, 1]
# print y_pred
p.append(np.ndarray.flatten(y_pred.data.cpu().numpy()))
p = [item for sublist in p for item in sublist]
p = np.array(p)
return p
def test_accuracy_full_batch(tokens, features, mini_batch_size, word_attn, sent_attn, th=0.5):
p = []
l = []
cnt = 0
g = gen_minibatch1(tokens, features, mini_batch_size, False)
for token, feature in g:
if cnt % 100 == 0:
print cnt
cnt +=1
# print token.size()
# y_pred = get_predictions(token, word_attn, sent_attn)
# print y_pred
y_pred = get_predictions(token, feature, word_attn, sent_attn)
# print y_pred
# _, y_pred = torch.max(y_pred, 1)
# y_pred = y_pred[:, 1]
# print y_pred
p.append(np.ndarray.flatten(y_pred.data.cpu().numpy()))
p = [item for sublist in p for item in sublist]
p = np.array(p)
return p
def _ncc_c(x, y):
"""
>>> _ncc_c([1,2,3,4], [1,2,3,4])
array([ 0.13333333, 0.36666667, 0.66666667, 1. , 0.66666667,
0.36666667, 0.13333333])
>>> _ncc_c([1,1,1], [1,1,1])
array([ 0.33333333, 0.66666667, 1. , 0.66666667, 0.33333333])
>>> _ncc_c([1,2,3], [-1,-1,-1])
array([-0.15430335, -0.46291005, -0.9258201 , -0.77151675, -0.46291005])
"""
den = np.array(norm(x) * norm(y))
den[den == 0] = np.Inf
x_len = len(x)
fft_size = 1<<(2*x_len-1).bit_length()
cc = ifft(fft(x, fft_size) * np.conj(fft(y, fft_size)))
cc = np.concatenate((cc[-(x_len-1):], cc[:x_len]))
return np.real(cc) / den
def layout_tree(correlation):
"""Layout tree for visualization with e.g. matplotlib.
Args:
correlation: A [V, V]-shaped numpy array of latent correlations.
Returns:
A [V, 3]-shaped numpy array of spectral positions of vertices.
"""
assert len(correlation.shape) == 2
assert correlation.shape[0] == correlation.shape[1]
assert correlation.dtype == np.float32
laplacian = -correlation
np.fill_diagonal(laplacian, 0)
np.fill_diagonal(laplacian, -laplacian.sum(axis=0))
evals, evects = scipy.linalg.eigh(laplacian, eigvals=[1, 2, 3])
assert np.all(evals > 0)
assert evects.shape[1] == 3
return evects
def __init__(self, N, V, tree_prior, config):
"""Initialize a model with an empty subsample.
Args:
N (int): Number of rows in the dataset.
V (int): Number of columns (features) in the dataset.
tree_prior: A [K]-shaped numpy array of prior edge log odds, where
K is the number of edges in the complete graph on V vertices.
config: A global config dict.
"""
assert isinstance(N, int)
assert isinstance(V, int)
assert isinstance(tree_prior, np.ndarray)
assert isinstance(config, dict)
K = V * (V - 1) // 2 # Number of edges in complete graph.
assert V <= 32768, 'Invalid # features > 32768: {}'.format(V)
assert tree_prior.shape == (K, )
assert tree_prior.dtype == np.float32
self._config = config.copy()
self._num_rows = N
self._tree_prior = tree_prior
self._tree = TreeStructure(V)
assert self._tree.num_vertices == V
self._program = make_propagation_program(self._tree.tree_grid)
self._added_rows = set()
def sample_tree(self):
"""Samples a random tree.
Returns:
A pair (edges, edge_logits), where:
edges: A list of (vertex, vertex) pairs.
edge_logits: A [K]-shaped numpy array of edge logits.
"""
logger.info('TreeCatTrainer.sample_tree given %d rows',
len(self._added_rows))
SERIES.sample_tree_num_rows.append(len(self._added_rows))
complete_grid = self._tree.complete_grid
edge_logits = self.compute_edge_logits()
assert edge_logits.shape[0] == complete_grid.shape[1]
assert edge_logits.dtype == np.float32
edges = self.get_edges()
edges = sample_tree(complete_grid, edge_logits, edges)
return edges, edge_logits
def compute_edge_logits(self):
"""Compute non-normalized logprob of all V(V-1)/2 candidate edges.
This is used for sampling and estimating the latent tree.
"""
V, E, K, M = self._VEKM
vert_logits = logprob_dc(self._vert_ss, self._vert_prior, axis=1)
if len(self._added_rows) == V:
assignments = self._assignments
else:
assignments = self._assignments[sorted(self._added_rows), :]
assignments = np.array(assignments, order='F')
parallel = self._config['learning_parallel']
result = treecat_compute_edge_logits(M, self._tree.complete_grid,
self._gammaln_table, assignments,
vert_logits, parallel)
result += self._tree_prior
return result
def train(self):
"""Train a TreeCat model using subsample-annealed MCMC.
Returns:
A trained model as a dictionary with keys:
config: A global config dict.
tree: A TreeStructure instance with the learned latent
structure.
edge_logits: A [K]-shaped array of all edge logits.
suffstats: Sufficient statistics of features, vertices, and
edges and a ragged_index for the features array.
assignments: An [N, V]-shaped numpy array of latent cluster
ids for each cell in the dataset, where N be the number of
data rows and V is the number of features.
"""
model = TreeTrainer.train(self)
model['assignments'] = self._assignments
model['suffstats'] = {
'ragged_index': self._table.ragged_index,
'vert_ss': self._vert_ss,
'edge_ss': self._edge_ss,
'feat_ss': self._feat_ss,
'meas_ss': self._meas_ss,
}
return model
def __init__(self, data, tree_prior, config):
"""Initialize a model with an empty subsample.
Args:
data: An [N, V]-shaped numpy array of real-valued data.
tree_prior: A [K]-shaped numpy array of prior edge log odds, where
K is the number of edges in the complete graph on V vertices.
config: A global config dict.
"""
assert isinstance(data, np.ndarray)
data = np.asarray(data, np.float32)
assert len(data.shape) == 2
N, V = data.shape
D = config['model_latent_dim']
E = V - 1 # Number of edges in the tree.
TreeTrainer.__init__(self, N, V, tree_prior, config)
self._data = data
self._latent = np.zeros([N, V, D], np.float32)
# This is symmetric positive definite.
self._vert_ss = np.zeros([V, D, D], np.float32)
# This is arbitrary (not necessarily symmetric).
self._edge_ss = np.zeros([E, D, D], np.float32)
# This represents (count, mean, covariance).
self._feat_ss = np.zeros([V, D, 1 + 1 + D], np.float32)
def train(self):
"""Train a TreeGauss model using subsample-annealed MCMC.
Returns:
A trained model as a dictionary with keys:
config: A global config dict.
tree: A TreeStructure instance with the learned latent
structure.
edge_logits: A [K]-shaped array of all edge logits.
suffstats: Sufficient statistics of features and vertices.
latent: An [N, V, M]-shaped numpy array of latent states, where
N is the number of data rows, V is the number of features,
and M is the dimension of each latent variable.
"""
model = TreeTrainer.train(self)
model['latent'] = self._latent
model['suffstats'] = {
'vert_ss': self._vert_ss,
'edge_ss': self._edge_ss,
'feat_ss': self._feat_ss,
}
return model
def train_ensemble(table, tree_prior, config):
"""Train a TreeCat ensemble model using subsample-annealed MCMC.
The ensemble size is controlled by config['model_ensemble_size'].
Let N be the number of data rows and V be the number of features.
Args:
table: A Table instance holding N rows of V features of data.
tree_prior: A [K]-shaped numpy array of prior edge log odds, where
K is the number of edges in the complete graph on V vertices.
config: A global config dict.
Returns:
A trained model as a dictionary with keys:
tree: A TreeStructure instance with the learned latent structure.
suffstats: Sufficient statistics of features, vertices, and edges.
assignments: An [N, V] numpy array of latent cluster ids for each
cell in the dataset.
"""
tasks = []
for sub_seed in range(config['model_ensemble_size']):
sub_config = config.copy()
sub_config['seed'] += sub_seed
tasks.append((table, tree_prior, sub_config))
return parallel_map(_train_model, tasks)
def test_server_logprob_normalized(N, V, C, M):
model = generate_fake_model(N, V, C, M)
config = TINY_CONFIG.copy()
config['model_num_clusters'] = M
model['config'] = config
server = TreeCatServer(model)
# The total probability of all categorical rows should be 1.
ragged_index = model['suffstats']['ragged_index']
factors = []
for v in range(V):
C = ragged_index[v + 1] - ragged_index[v]
factors.append([one_hot(c, C) for c in range(C)])
data = np.array(
[np.concatenate(columns) for columns in itertools.product(*factors)],
dtype=np.int8)
logprobs = server.logprob(data)
logtotal = np.logaddexp.reduce(logprobs)
assert logtotal == pytest.approx(0.0, abs=1e-5)
def observed_perplexity(self, counts):
"""Compute perplexity = exp(entropy) of observed variables.
Perplexity is an information theoretic measure of the number of
clusters or latent classes. Perplexity is a real number in the range
[1, M], where M is model_num_clusters.
Args:
counts: A [V]-shaped array of multinomial counts.
Returns:
A [V]-shaped numpy array of perplexity.
"""
V, E, M, R = self._VEMR
if counts is not None:
counts = np.ones(V, dtype=np.int8)
assert counts.shape == (V, )
assert counts.dtype == np.int8
assert np.all(counts > 0)
observed_entropy = np.empty(V, dtype=np.float32)
for v in range(V):
beg, end = self._ragged_index[v:v + 2]
probs = np.dot(self._feat_cond[beg:end, :], self._vert_probs[v, :])
observed_entropy[v] = multinomial_entropy(probs, counts[v])
return np.exp(observed_entropy)
def observed_perplexity(self, counts):
"""Compute perplexity = exp(entropy) of observed variables.
Perplexity is an information theoretic measure of the number of
clusters or observed classes. Perplexity is a real number in the range
[1, dim[v]], where dim[v] is the number of categories in an observed
categorical variable or 2 for an ordinal variable.
Args:
counts: A [V]-shaped array of multinomial counts.
Returns:
A [V]-shaped numpy array of perplexity.
"""
result = self._ensemble[0].observed_perplexity(counts)
for server in self._ensemble[1:]:
result += server.observed_perplexity(counts)
result /= len(self._ensemble)
return result
def latent_correlation(self):
"""Compute correlation matrix among latent features.
This computes the generalization of Pearson's correlation to discrete
data. Let I(X;Y) be the mutual information. Then define correlation as
rho(X,Y) = sqrt(1 - exp(-2 I(X;Y)))
Returns:
A [V, V]-shaped numpy array of feature-feature correlations.
"""
result = self._ensemble[0].latent_correlation()
for server in self._ensemble[1:]:
result += server.latent_correlation()
result /= len(self._ensemble)
return result
def logprob(self, rows, evidence=None):
"""Compute non-normalized log probabilies of many rows of data.
If evidence is specified, compute conditional log probability;
otherwise compute unconditional log probability.
Args:
data: A list of rows of data, where each row is a sparse dict
mapping feature name to feature value.
evidence: An optional row of conditioning data, as a sparse dict
mapping feature name to feature value.
Returns:
An [len(rows)]-shaped numpy array of log probabilities.
"""
data = import_rows(self._schema, rows)
if evidence is None:
return self._server.logprob(data)
else:
ragged_evidence = import_rows(self._schema, [evidence])
return (self._server.logprob(data + ragged_evidence) -
self._server.logprob(data + evidence))
def sample(self, N, evidence=None):
"""Draw N samples from the posterior distribution.
Args:
N: The number of samples to draw.
evidence: An optional single row of conditioning data, as a sparse
dict mapping feature name to feature value.
Returns:
An [N, R]-shaped numpy array of sampled multinomial data.
"""
if evidence is None:
data = None
else:
data = import_rows(self._schema, [evidence])[0]
ragged_samples = self._server.sample(N, self._counts, data)
return export_rows(self._schema, ragged_samples)
def fit(self, X, y):
""" Training the gcForest on input data X and associated target y.
:param X: np.array
Array containing the input samples.
Must be of shape [n_samples, data] where data is a 1D array.
:param y: np.array
1D array containing the target values.
Must be of shape [n_samples]
"""
if np.shape(X)[0] != len(y):
raise ValueError('Sizes of y and X do not match.')
mgs_X = self.mg_scanning(X, y)
_ = self.cascade_forest(mgs_X, y)
def contest(self, b, g, r):
""" Search for biased BGR values
Finds closest neuron (min dist) and updates self.freq
finds best neuron (min dist-self.bias) and returns position
for frequently chosen neurons, self.freq[i] is high and self.bias[i] is negative
self.bias[i] = self.GAMMA*((1/self.NETSIZE)-self.freq[i])"""
i, j = self.SPECIALS, self.NETSIZE
dists = abs(self.network[i:j] - np.array([b,g,r])).sum(1)
bestpos = i + np.argmin(dists)
biasdists = dists - self.bias[i:j]
bestbiaspos = i + np.argmin(biasdists)
self.freq[i:j] *= (1-self.BETA)
self.bias[i:j] += self.BETAGAMMA * self.freq[i:j]
self.freq[bestpos] += self.BETA
self.bias[bestpos] -= self.BETAGAMMA
return bestbiaspos
def rasterMaskToGrid( rasterMask ):
grid = []
mask = rasterMask['mask']
for y in range(rasterMask['height']):
for x in range(rasterMask['width']):
if mask[y,x]==0:
grid.append([x,y])
grid = np.array(grid,dtype=np.float)
if not (rasterMask is None) and rasterMask['hex'] is True:
f = math.sqrt(3.0)/2.0
offset = -0.5
if np.argmin(rasterMask['mask'][0]) > np.argmin(rasterMask['mask'][1]):
offset = 0.5
for i in range(len(grid)):
if (grid[i][1]%2.0==0.0):
grid[i][0]-=offset
grid[i][1] *= f
return grid
def match_matrix(event: Event):
"""Returns a numpy participation matrix for the qualification matches in this event, used for calculating OPR.
Each row in the matrix corresponds to a single alliance in a match, meaning that there will be two rows (one for
red, one for blue) per match. Each column represents a single team, ordered by team number. If a team participated
on a certain alliance, the value at that row and column would be 1, otherwise, it would be 0. For example, an
event with teams 1-7 that featured a match that pitted teams 1, 3, and 5 against 2, 4, and 6 would have a match
matrix that looks like this (sans labels):
#1 #2 #3 #4 #5 #6 #7
qm1_red 1 0 1 0 1 0 0
qm1_blue 0 1 0 1 0 1 0
"""
match_list = []
for match in filter(lambda match: match['comp_level'] == 'qm', event.matches):
matchRow = []
for team in event.teams:
matchRow.append(1 if team['key'] in match['alliances']['red']['teams'] else 0)
match_list.append(matchRow)
matchRow = []
for team in event.teams:
matchRow.append(1 if team['key'] in match['alliances']['blue']['teams'] else 0)
match_list.append(matchRow)
mat = numpy.array(match_list)
sum_matches = numpy.sum(mat, axis=0)
avg_team_matches = sum(sum_matches) / float(len(sum_matches))
return mat[:, numpy.apply_along_axis(numpy.count_nonzero, 0, mat) > avg_team_matches - 2]
def get_img(data_path):
# Getting image array from path:
img = imread(data_path)
img = imresize(img, (64, 64))
return img
def get_dataset(dataset_path='Data/Train_Data'):
# Getting all data from data path:
try:
X = np.load('Data/npy_train_data/X.npy')
Y = np.load('Data/npy_train_data/Y.npy')
except:
inputs_path = dataset_path+'/input'
images = listdir(inputs_path) # Geting images
X = []
Y = []
for img in images:
img_path = inputs_path+'/'+img
x_img = get_img(img_path).astype('float32').reshape(64, 64, 3)
x_img /= 255.
y_img = get_img(img_path.replace('input/', 'mask/mask_')).astype('float32').reshape(64, 64, 1)
y_img /= 255.
X.append(x_img)
Y.append(y_img)
X = np.array(X)
Y = np.array(Y)
# Create dateset:
if not os.path.exists('Data/npy_train_data/'):
os.makedirs('Data/npy_train_data/')
np.save('Data/npy_train_data/X.npy', X)
np.save('Data/npy_train_data/Y.npy', Y)
X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42)
return X, X_test, Y, Y_test
def read_groundtruth():
ret = []
with open(
os.path.join(
os.path.abspath(os.path.dirname(__file__)),
'groundtruth.txt'), 'rb') as lines:
for line in lines:
ret.append(line[:-2])
return np.array(ret)
def extract_digits(self, image):
"""
Extract digits from a binary image representing a sudoku
:param image: binary image/sudoku
:return: array of digits and their probabilities
"""
prob = np.zeros(4, dtype=np.float32)
digits = np.zeros((4, 9, 9), dtype=object)
for i in range(4):
labeled, features = label(image, structure=CROSS)
objs = find_objects(labeled)
for obj in objs:
roi = image[obj]
# center of bounding box
cy = (obj[0].stop + obj[0].start) / 2
cx = (obj[1].stop + obj[1].start) / 2
dists = cdist([[cy, cx]], CENTROIDS, 'euclidean')
pos = np.argmin(dists)
cy, cx = pos % 9, pos / 9
# 28x28 image, center relative to sudoku
prediction = self.classifier.classify(morph(roi))
if digits[i, cy, cx] is 0:
# Newly found digit
digits[i, cy, cx] = prediction
prob[i] += prediction[0, 0]
elif prediction[0, 0] > digits[i, cy, cx][0, 0]:
# Overlapping! (noise), choose the most probable prediction
prob[i] -= digits[i, cy, cx][0, 0]
digits[i, cy, cx] = prediction
prob[i] += prediction[0, 0]
image = np.rot90(image)
logging.info(prob)
return digits[np.argmax(prob)]
def diagonal(_, pos):
"""
Given an object pixels' positions, return the diagonal length of its
bound box
:param _: pixel values (unused)
:param pos: pixel position (1-D)
:return: diagonal of bounding box
"""
xs = np.array([i / SSIZE for i in pos])
ys = np.array([i % SSIZE for i in pos])
minx = np.amin(xs)
miny = np.amin(ys)
maxx = np.amax(xs)
maxy = np.amax(ys)
return compute_line(np.array([minx, miny]), np.array([maxx, maxy]))