Python pandas 模块，option_context() 实例源码
我们从Python开源项目中，提取了以下42个代码示例，用于说明如何使用pandas.option_context()。
def summary(self):
"""
Print the summary of the Process model.
:return: None
"""
with pd.option_context("precision", self.options["precision"]):
full_model = self.outcome_models[self.iv]
m_models = [self.outcome_models.get(med_name) for med_name in self.mediators]
if self.options["detail"]:
print("\n***************************** OUTCOME MODELS ****************************\n")
print(full_model)
print("\n-------------------------------------------------------------------------\n")
for med_model in m_models:
print(med_model)
print("\n-------------------------------------------------------------------------\n")
if self.indirect_model:
print("\n********************** DIRECT AND INDIRECT EFFECTS **********************\n")
print(self.direct_model)
print(self.indirect_model)
else:
print("\n********************** CONDITIONAL EFFECTS **********************\n")
print(self.direct_model)
def test_show_null_counts(self):
df = DataFrame(1, columns=range(10), index=range(10))
df.iloc[1, 1] = np.nan
def check(null_counts, result):
buf = StringIO()
df.info(buf=buf, null_counts=null_counts)
self.assertTrue(('non-null' in buf.getvalue()) is result)
with option_context('display.max_info_rows', 20,
'display.max_info_columns', 20):
check(None, True)
check(True, True)
check(False, False)
with option_context('display.max_info_rows', 5,
'display.max_info_columns', 5):
check(None, False)
check(True, False)
check(False, False)
def test_repr_truncation(self):
max_len = 20
with option_context("display.max_colwidth", max_len):
df = DataFrame({'A': np.random.randn(10),
'B': [tm.rands(np.random.randint(
max_len - 1, max_len + 1)) for i in range(10)
]})
r = repr(df)
r = r[r.find('\n') + 1:]
adj = fmt._get_adjustment()
for line, value in lzip(r.split('\n'), df['B']):
if adj.len(value) + 1 > max_len:
self.assertIn('...', line)
else:
self.assertNotIn('...', line)
with option_context("display.max_colwidth", 999999):
self.assertNotIn('...', repr(df))
with option_context("display.max_colwidth", max_len + 2):
self.assertNotIn('...', repr(df))
def test_expand_frame_repr(self):
df_small = DataFrame('hello', [0], [0])
df_wide = DataFrame('hello', [0], lrange(10))
df_tall = DataFrame('hello', lrange(30), lrange(5))
with option_context('mode.sim_interactive', True):
with option_context('display.max_columns', 10, 'display.width', 20,
'display.max_rows', 20,
'display.show_dimensions', True):
with option_context('display.expand_frame_repr', True):
self.assertFalse(has_truncated_repr(df_small))
self.assertFalse(has_expanded_repr(df_small))
self.assertFalse(has_truncated_repr(df_wide))
self.assertTrue(has_expanded_repr(df_wide))
self.assertTrue(has_vertically_truncated_repr(df_tall))
self.assertTrue(has_expanded_repr(df_tall))
with option_context('display.expand_frame_repr', False):
self.assertFalse(has_truncated_repr(df_small))
self.assertFalse(has_expanded_repr(df_small))
self.assertFalse(has_horizontally_truncated_repr(df_wide))
self.assertFalse(has_expanded_repr(df_wide))
self.assertTrue(has_vertically_truncated_repr(df_tall))
self.assertFalse(has_expanded_repr(df_tall))
def test_str_max_colwidth(self):
# GH 7856
df = pd.DataFrame([{'a': 'foo',
'b': 'bar',
'c': 'uncomfortably long line with lots of stuff',
'd': 1}, {'a': 'foo',
'b': 'bar',
'c': 'stuff',
'd': 1}])
df.set_index(['a', 'b', 'c'])
self.assertTrue(
str(df) ==
' a b c d\n'
'0 foo bar uncomfortably long line with lots of stuff 1\n'
'1 foo bar stuff 1')
with option_context('max_colwidth', 20):
self.assertTrue(str(df) == ' a b c d\n'
'0 foo bar uncomfortably lo... 1\n'
'1 foo bar stuff 1')
def test_wide_repr(self):
with option_context('mode.sim_interactive', True,
'display.show_dimensions', True):
max_cols = get_option('display.max_columns')
df = DataFrame(tm.rands_array(25, size=(10, max_cols - 1)))
set_option('display.expand_frame_repr', False)
rep_str = repr(df)
assert "10 rows x %d columns" % (max_cols - 1) in rep_str
set_option('display.expand_frame_repr', True)
wide_repr = repr(df)
self.assertNotEqual(rep_str, wide_repr)
with option_context('display.width', 120):
wider_repr = repr(df)
self.assertTrue(len(wider_repr) < len(wide_repr))
reset_option('display.expand_frame_repr')
def test_wide_repr_named(self):
with option_context('mode.sim_interactive', True):
max_cols = get_option('display.max_columns')
df = DataFrame(tm.rands_array(25, size=(10, max_cols - 1)))
df.index.name = 'DataFrame Index'
set_option('display.expand_frame_repr', False)
rep_str = repr(df)
set_option('display.expand_frame_repr', True)
wide_repr = repr(df)
self.assertNotEqual(rep_str, wide_repr)
with option_context('display.width', 150):
wider_repr = repr(df)
self.assertTrue(len(wider_repr) < len(wide_repr))
for line in wide_repr.splitlines()[1::13]:
self.assertIn('DataFrame Index', line)
reset_option('display.expand_frame_repr')
def test_wide_repr_multiindex_cols(self):
with option_context('mode.sim_interactive', True):
max_cols = get_option('display.max_columns')
midx = MultiIndex.from_arrays(tm.rands_array(5, size=(2, 10)))
mcols = MultiIndex.from_arrays(tm.rands_array(3, size=(2, max_cols
- 1)))
df = DataFrame(tm.rands_array(25, (10, max_cols - 1)),
index=midx, columns=mcols)
df.index.names = ['Level 0', 'Level 1']
set_option('display.expand_frame_repr', False)
rep_str = repr(df)
set_option('display.expand_frame_repr', True)
wide_repr = repr(df)
self.assertNotEqual(rep_str, wide_repr)
with option_context('display.width', 150):
wider_repr = repr(df)
self.assertTrue(len(wider_repr) < len(wide_repr))
reset_option('display.expand_frame_repr')
def test_show_dimensions(self):
df = DataFrame(123, lrange(10, 15), lrange(30))
with option_context('display.max_rows', 10, 'display.max_columns', 40,
'display.width', 500, 'display.expand_frame_repr',
'info', 'display.show_dimensions', True):
self.assertTrue('5 rows' in str(df))
self.assertTrue('5 rows' in df._repr_html_())
with option_context('display.max_rows', 10, 'display.max_columns', 40,
'display.width', 500, 'display.expand_frame_repr',
'info', 'display.show_dimensions', False):
self.assertFalse('5 rows' in str(df))
self.assertFalse('5 rows' in df._repr_html_())
with option_context('display.max_rows', 2, 'display.max_columns', 2,
'display.width', 500, 'display.expand_frame_repr',
'info', 'display.show_dimensions', 'truncate'):
self.assertTrue('5 rows' in str(df))
self.assertTrue('5 rows' in df._repr_html_())
with option_context('display.max_rows', 10, 'display.max_columns', 40,
'display.width', 500, 'display.expand_frame_repr',
'info', 'display.show_dimensions', 'truncate'):
self.assertFalse('5 rows' in str(df))
self.assertFalse('5 rows' in df._repr_html_())
def test_info_repr(self):
max_rows = get_option('display.max_rows')
max_cols = get_option('display.max_columns')
# Long
h, w = max_rows + 1, max_cols - 1
df = DataFrame(dict((k, np.arange(1, 1 + h)) for k in np.arange(w)))
assert has_vertically_truncated_repr(df)
with option_context('display.large_repr', 'info'):
assert has_info_repr(df)
# Wide
h, w = max_rows - 1, max_cols + 1
df = DataFrame(dict((k, np.arange(1, 1 + h)) for k in np.arange(w)))
assert has_horizontally_truncated_repr(df)
with option_context('display.large_repr', 'info'):
assert has_info_repr(df)
def test_info_repr_html(self):
max_rows = get_option('display.max_rows')
max_cols = get_option('display.max_columns')
# Long
h, w = max_rows + 1, max_cols - 1
df = DataFrame(dict((k, np.arange(1, 1 + h)) for k in np.arange(w)))
assert r'<class' not in df._repr_html_()
with option_context('display.large_repr', 'info'):
assert r'<class' in df._repr_html_()
# Wide
h, w = max_rows - 1, max_cols + 1
df = DataFrame(dict((k, np.arange(1, 1 + h)) for k in np.arange(w)))
assert '<class' not in df._repr_html_()
with option_context('display.large_repr', 'info'):
assert '<class' in df._repr_html_()
def test_format_explicit(self):
test_sers = self.gen_test_series()
with option_context("display.max_rows", 4):
res = repr(test_sers['onel'])
exp = '0 a\n1 a\n ..\n98 a\n99 a\ndtype: object'
self.assertEqual(exp, res)
res = repr(test_sers['twol'])
exp = ('0 ab\n1 ab\n ..\n98 ab\n99 ab\ndtype:'
' object')
self.assertEqual(exp, res)
res = repr(test_sers['asc'])
exp = ('0 a\n1 ab\n ... \n4 abcde\n5'
' abcdef\ndtype: object')
self.assertEqual(exp, res)
res = repr(test_sers['desc'])
exp = ('5 abcdef\n4 abcde\n ... \n1 ab\n0'
' a\ndtype: object')
self.assertEqual(exp, res)
def df_to_html(df, border=0, classes=('table', 'table-striped', 'table-hover'),
**kwargs):
"""Convert a dataframe to HTML without truncating contents.
pandas will truncate cell contents that exceed 50 characters by default.
Use this function to avoid this truncation behavior.
This function uses different default parameters than `DataFrame.to_html` to
give uniform styling to HTML tables that are compatible with q2template
themes. These parameters can be overridden, and they (along with any other
parameters) will be passed through to `DataFrame.to_html`.
Parameters
----------
df : pd.DataFrame
DataFrame to convert to HTML.
kwargs : dict
Parameters passed through to `pd.DataFrame.to_html`.
Returns
-------
str
DataFrame converted to HTML.
References
----------
.. [1] https://stackoverflow.com/q/26277757/3776794
.. [2] https://github.com/pandas-dev/pandas/issues/1852
"""
with pd.option_context('display.max_colwidth', -1):
return df.to_html(border=border, classes=classes, **kwargs)
def test_representation(self):
idx = []
idx.append(DatetimeIndex([], freq='D'))
idx.append(DatetimeIndex(['2011-01-01'], freq='D'))
idx.append(DatetimeIndex(['2011-01-01', '2011-01-02'], freq='D'))
idx.append(DatetimeIndex(
['2011-01-01', '2011-01-02', '2011-01-03'], freq='D'))
idx.append(DatetimeIndex(
['2011-01-01 09:00', '2011-01-01 10:00', '2011-01-01 11:00'
], freq='H', tz='Asia/Tokyo'))
idx.append(DatetimeIndex(
['2011-01-01 09:00', '2011-01-01 10:00', pd.NaT], tz='US/Eastern'))
idx.append(DatetimeIndex(
['2011-01-01 09:00', '2011-01-01 10:00', pd.NaT], tz='UTC'))
exp = []
exp.append("""DatetimeIndex([], dtype='datetime64[ns]', freq='D')""")
exp.append("DatetimeIndex(['2011-01-01'], dtype='datetime64[ns]', "
"freq='D')")
exp.append("DatetimeIndex(['2011-01-01', '2011-01-02'], "
"dtype='datetime64[ns]', freq='D')")
exp.append("DatetimeIndex(['2011-01-01', '2011-01-02', '2011-01-03'], "
"dtype='datetime64[ns]', freq='D')")
exp.append("DatetimeIndex(['2011-01-01 09:00:00+09:00', "
"'2011-01-01 10:00:00+09:00', '2011-01-01 11:00:00+09:00']"
", dtype='datetime64[ns, Asia/Tokyo]', freq='H')")
exp.append("DatetimeIndex(['2011-01-01 09:00:00-05:00', "
"'2011-01-01 10:00:00-05:00', 'NaT'], "
"dtype='datetime64[ns, US/Eastern]', freq=None)")
exp.append("DatetimeIndex(['2011-01-01 09:00:00+00:00', "
"'2011-01-01 10:00:00+00:00', 'NaT'], "
"dtype='datetime64[ns, UTC]', freq=None)""")
with pd.option_context('display.width', 300):
for indx, expected in zip(idx, exp):
for func in ['__repr__', '__unicode__', '__str__']:
result = getattr(indx, func)()
self.assertEqual(result, expected)
def test_representation(self):
idx1 = TimedeltaIndex([], freq='D')
idx2 = TimedeltaIndex(['1 days'], freq='D')
idx3 = TimedeltaIndex(['1 days', '2 days'], freq='D')
idx4 = TimedeltaIndex(['1 days', '2 days', '3 days'], freq='D')
idx5 = TimedeltaIndex(['1 days 00:00:01', '2 days', '3 days'])
exp1 = """TimedeltaIndex([], dtype='timedelta64[ns]', freq='D')"""
exp2 = ("TimedeltaIndex(['1 days'], dtype='timedelta64[ns]', "
"freq='D')")
exp3 = ("TimedeltaIndex(['1 days', '2 days'], "
"dtype='timedelta64[ns]', freq='D')")
exp4 = ("TimedeltaIndex(['1 days', '2 days', '3 days'], "
"dtype='timedelta64[ns]', freq='D')")
exp5 = ("TimedeltaIndex(['1 days 00:00:01', '2 days 00:00:00', "
"'3 days 00:00:00'], dtype='timedelta64[ns]', freq=None)")
with pd.option_context('display.width', 300):
for idx, expected in zip([idx1, idx2, idx3, idx4, idx5],
[exp1, exp2, exp3, exp4, exp5]):
for func in ['__repr__', '__unicode__', '__str__']:
result = getattr(idx, func)()
self.assertEqual(result, expected)
def test_representation_to_series(self):
idx1 = TimedeltaIndex([], freq='D')
idx2 = TimedeltaIndex(['1 days'], freq='D')
idx3 = TimedeltaIndex(['1 days', '2 days'], freq='D')
idx4 = TimedeltaIndex(['1 days', '2 days', '3 days'], freq='D')
idx5 = TimedeltaIndex(['1 days 00:00:01', '2 days', '3 days'])
exp1 = """Series([], dtype: timedelta64[ns])"""
exp2 = """0 1 days
dtype: timedelta64[ns]"""
exp3 = """0 1 days
1 2 days
dtype: timedelta64[ns]"""
exp4 = """0 1 days
1 2 days
2 3 days
dtype: timedelta64[ns]"""
exp5 = """0 1 days 00:00:01
1 2 days 00:00:00
2 3 days 00:00:00
dtype: timedelta64[ns]"""
with pd.option_context('display.width', 300):
for idx, expected in zip([idx1, idx2, idx3, idx4, idx5],
[exp1, exp2, exp3, exp4, exp5]):
result = repr(pd.Series(idx))
self.assertEqual(result, expected)
def test_repr_max_seq_item_setting(self):
# GH10182
idx = self.create_index()
idx = idx.repeat(50)
with pd.option_context("display.max_seq_items", None):
repr(idx)
self.assertFalse('...' in str(idx))
def test_isnull_for_inf(self):
s = Series(['a', np.inf, np.nan, 1.0])
with pd.option_context('mode.use_inf_as_null', True):
r = s.isnull()
dr = s.dropna()
e = Series([False, True, True, False])
de = Series(['a', 1.0], index=[0, 3])
tm.assert_series_equal(r, e)
tm.assert_series_equal(dr, de)
def test_repr_chop_threshold(self):
df = DataFrame([[0.1, 0.5], [0.5, -0.1]])
pd.reset_option("display.chop_threshold") # default None
self.assertEqual(repr(df), ' 0 1\n0 0.1 0.5\n1 0.5 -0.1')
with option_context("display.chop_threshold", 0.2):
self.assertEqual(repr(df), ' 0 1\n0 0.0 0.5\n1 0.5 0.0')
with option_context("display.chop_threshold", 0.6):
self.assertEqual(repr(df), ' 0 1\n0 0.0 0.0\n1 0.0 0.0')
with option_context("display.chop_threshold", None):
self.assertEqual(repr(df), ' 0 1\n0 0.1 0.5\n1 0.5 -0.1')
def test_repr_obeys_max_seq_limit(self):
with option_context("display.max_seq_items", 2000):
self.assertTrue(len(com.pprint_thing(lrange(1000))) > 1000)
with option_context("display.max_seq_items", 5):
self.assertTrue(len(com.pprint_thing(lrange(1000))) < 100)
def test_repr_non_interactive(self):
# in non interactive mode, there can be no dependency on the
# result of terminal auto size detection
df = DataFrame('hello', lrange(1000), lrange(5))
with option_context('mode.sim_interactive', False, 'display.width', 0,
'display.height', 0, 'display.max_rows', 5000):
self.assertFalse(has_truncated_repr(df))
self.assertFalse(has_expanded_repr(df))
def test_auto_detect(self):
term_width, term_height = get_terminal_size()
fac = 1.05 # Arbitrary large factor to exceed term widht
cols = range(int(term_width * fac))
index = range(10)
df = DataFrame(index=index, columns=cols)
with option_context('mode.sim_interactive', True):
with option_context('max_rows', None):
with option_context('max_columns', None):
# Wrap around with None
self.assertTrue(has_expanded_repr(df))
with option_context('max_rows', 0):
with option_context('max_columns', 0):
# Truncate with auto detection.
self.assertTrue(has_horizontally_truncated_repr(df))
index = range(int(term_height * fac))
df = DataFrame(index=index, columns=cols)
with option_context('max_rows', 0):
with option_context('max_columns', None):
# Wrap around with None
self.assertTrue(has_expanded_repr(df))
# Truncate vertically
self.assertTrue(has_vertically_truncated_repr(df))
with option_context('max_rows', None):
with option_context('max_columns', 0):
self.assertTrue(has_horizontally_truncated_repr(df))
def test_to_string_truncate_indices(self):
for index in [tm.makeStringIndex, tm.makeUnicodeIndex, tm.makeIntIndex,
tm.makeDateIndex, tm.makePeriodIndex]:
for column in [tm.makeStringIndex]:
for h in [10, 20]:
for w in [10, 20]:
with option_context("display.expand_frame_repr",
False):
df = DataFrame(index=index(h), columns=column(w))
with option_context("display.max_rows", 15):
if h == 20:
self.assertTrue(
has_vertically_truncated_repr(df))
else:
self.assertFalse(
has_vertically_truncated_repr(df))
with option_context("display.max_columns", 15):
if w == 20:
self.assertTrue(
has_horizontally_truncated_repr(df))
else:
self.assertFalse(
has_horizontally_truncated_repr(df))
with option_context("display.max_rows", 15,
"display.max_columns", 15):
if h == 20 and w == 20:
self.assertTrue(has_doubly_truncated_repr(
df))
else:
self.assertFalse(has_doubly_truncated_repr(
df))
def test_to_string_truncate_multilevel(self):
arrays = [['bar', 'bar', 'baz', 'baz', 'foo', 'foo', 'qux', 'qux'],
['one', 'two', 'one', 'two', 'one', 'two', 'one', 'two']]
df = DataFrame(index=arrays, columns=arrays)
with option_context("display.max_rows", 7, "display.max_columns", 7):
self.assertTrue(has_doubly_truncated_repr(df))
def test_wide_repr_wide_columns(self):
with option_context('mode.sim_interactive', True):
df = DataFrame(randn(5, 3), columns=['a' * 90, 'b' * 90, 'c' * 90])
rep_str = repr(df)
self.assertEqual(len(rep_str.splitlines()), 20)
def test_wide_repr_unicode(self):
with option_context('mode.sim_interactive', True):
max_cols = get_option('display.max_columns')
df = DataFrame(tm.rands_array(25, size=(10, max_cols - 1)))
set_option('display.expand_frame_repr', False)
rep_str = repr(df)
set_option('display.expand_frame_repr', True)
wide_repr = repr(df)
self.assertNotEqual(rep_str, wide_repr)
with option_context('display.width', 150):
wider_repr = repr(df)
self.assertTrue(len(wider_repr) < len(wide_repr))
reset_option('display.expand_frame_repr')
def test_wide_repr_wide_long_columns(self):
with option_context('mode.sim_interactive', True):
df = DataFrame({'a': ['a' * 30, 'b' * 30],
'b': ['c' * 70, 'd' * 80]})
result = repr(df)
self.assertTrue('ccccc' in result)
self.assertTrue('ddddd' in result)
def test_max_multi_index_display(self):
# GH 7101
# doc example (indexing.rst)
# multi-index
arrays = [['bar', 'bar', 'baz', 'baz', 'foo', 'foo', 'qux', 'qux'],
['one', 'two', 'one', 'two', 'one', 'two', 'one', 'two']]
tuples = list(zip(*arrays))
index = MultiIndex.from_tuples(tuples, names=['first', 'second'])
s = Series(randn(8), index=index)
with option_context("display.max_rows", 10):
self.assertEqual(len(str(s).split('\n')), 10)
with option_context("display.max_rows", 3):
self.assertEqual(len(str(s).split('\n')), 5)
with option_context("display.max_rows", 2):
self.assertEqual(len(str(s).split('\n')), 5)
with option_context("display.max_rows", 1):
self.assertEqual(len(str(s).split('\n')), 4)
with option_context("display.max_rows", 0):
self.assertEqual(len(str(s).split('\n')), 10)
# index
s = Series(randn(8), None)
with option_context("display.max_rows", 10):
self.assertEqual(len(str(s).split('\n')), 9)
with option_context("display.max_rows", 3):
self.assertEqual(len(str(s).split('\n')), 4)
with option_context("display.max_rows", 2):
self.assertEqual(len(str(s).split('\n')), 4)
with option_context("display.max_rows", 1):
self.assertEqual(len(str(s).split('\n')), 3)
with option_context("display.max_rows", 0):
self.assertEqual(len(str(s).split('\n')), 9)
# Make sure #8532 is fixed
def test_consistent_format(self):
s = pd.Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.9999, 1, 1] * 10)
with option_context("display.max_rows", 10):
res = repr(s)
exp = ('0 1.0000\n1 1.0000\n2 1.0000\n3 '
'1.0000\n4 1.0000\n ... \n125 '
'1.0000\n126 1.0000\n127 0.9999\n128 '
'1.0000\n129 1.0000\ndtype: float64')
self.assertEqual(res, exp)
def chck_ncols(self, s):
with option_context("display.max_rows", 10):
res = repr(s)
lines = res.split('\n')
lines = [line for line in repr(s).split('\n')
if not re.match('[^\.]*\.+', line)][:-1]
ncolsizes = len(set(len(line.strip()) for line in lines))
self.assertEqual(ncolsizes, 1)
def test_truncate_ndots(self):
def getndots(s):
return len(re.match('[^\.]*(\.*)', s).groups()[0])
s = Series([0, 2, 3, 6])
with option_context("display.max_rows", 2):
strrepr = repr(s).replace('\n', '')
self.assertEqual(getndots(strrepr), 2)
s = Series([0, 100, 200, 400])
with option_context("display.max_rows", 2):
strrepr = repr(s).replace('\n', '')
self.assertEqual(getndots(strrepr), 3)
def test_output_significant_digits(self):
# Issue #9764
# In case default display precision changes:
with pd.option_context('display.precision', 6):
# DataFrame example from issue #9764
d = pd.DataFrame(
{'col1': [9.999e-8, 1e-7, 1.0001e-7, 2e-7, 4.999e-7, 5e-7,
5.0001e-7, 6e-7, 9.999e-7, 1e-6, 1.0001e-6, 2e-6,
4.999e-6, 5e-6, 5.0001e-6, 6e-6]})
expected_output = {
(0, 6):
' col1\n0 9.999000e-08\n1 1.000000e-07\n2 1.000100e-07\n3 2.000000e-07\n4 4.999000e-07\n5 5.000000e-07',
(1, 6):
' col1\n1 1.000000e-07\n2 1.000100e-07\n3 2.000000e-07\n4 4.999000e-07\n5 5.000000e-07',
(1, 8):
' col1\n1 1.000000e-07\n2 1.000100e-07\n3 2.000000e-07\n4 4.999000e-07\n5 5.000000e-07\n6 5.000100e-07\n7 6.000000e-07',
(8, 16):
' col1\n8 9.999000e-07\n9 1.000000e-06\n10 1.000100e-06\n11 2.000000e-06\n12 4.999000e-06\n13 5.000000e-06\n14 5.000100e-06\n15 6.000000e-06',
(9, 16):
' col1\n9 0.000001\n10 0.000001\n11 0.000002\n12 0.000005\n13 0.000005\n14 0.000005\n15 0.000006'
}
for (start, stop), v in expected_output.items():
self.assertEqual(str(d[start:stop]), v)
def test_too_long(self):
# GH 10451
with pd.option_context('display.precision', 4):
# need both a number > 1e6 and something that normally formats to
# having length > display.precision + 6
df = pd.DataFrame(dict(x=[12345.6789]))
self.assertEqual(str(df), ' x\n0 12345.6789')
df = pd.DataFrame(dict(x=[2e6]))
self.assertEqual(str(df), ' x\n0 2000000.0')
df = pd.DataFrame(dict(x=[12345.6789, 2e6]))
self.assertEqual(
str(df), ' x\n0 1.2346e+04\n1 2.0000e+06')
def test_precision(self):
with pd.option_context('display.precision', 10):
s = Styler(self.df)
self.assertEqual(s.precision, 10)
s = Styler(self.df, precision=2)
self.assertEqual(s.precision, 2)
s2 = s.set_precision(4)
self.assertTrue(s is s2)
self.assertEqual(s.precision, 4)
def _main(log_path, show_browser=False):
print(log_path)
df, metadata = process_log(log_path)
del df['Timestamp']
df['Msg Type'] = df['Msg Type'].apply(escape_html_chars)
df['Message'] = df['Message'].apply(escape_html_chars)
# df['Message'] = df['Message'].apply(try_json)
df['Message'] = df.apply(lambda row: format_error(row['Msg Type'], row['Message']), 1)
df['Rev ID'] = df['Rev ID'].apply(lambda x: '<a href="https://www.wikidata.org/w/index.php?oldid={}&diff=prev">{}</a>'.format(x,x) if x else x)
level_counts, info_counts, warning_counts, error_counts = generate_summary(df)
warnings_df = df.query("Level == 'WARNING'")
warnings_df.is_copy = False
del warnings_df['Level']
if not warnings_df.empty:
warnings_df = gen_ext_id_links(warnings_df)
warnings_df = url_qid(warnings_df, "QID")
errors_df = df.query("Level == 'ERROR'")
errors_df.is_copy = False
del errors_df['Level']
if not errors_df.empty:
errors_df = gen_ext_id_links(errors_df)
errors_df = url_qid(errors_df, "QID")
# errors_df['Message'] = errors_df['Message'].apply(try_format_error)
info_df = df.query("Level == 'INFO'")
info_df.is_copy = False
del info_df['Level']
if not info_df.empty:
info_df = gen_ext_id_links(info_df)
info_df = url_qid(info_df, "QID")
info_df.Message = info_df.Message.str.replace("SKIP", "No Action")
with pd.option_context('display.max_colwidth', -1):
# this class nonsense is an ugly hack: https://stackoverflow.com/questions/15079118/js-datatables-from-pandas/41536906
level_counts = level_counts.to_frame().to_html(escape=False)
info_counts = info_counts.to_frame().to_html(escape=False)
warning_counts = warning_counts.to_frame().to_html(escape=False)
error_counts = error_counts.to_frame().to_html(escape=False)
info_df = info_df.to_html(escape=False, classes='df" id = "info_df')
warnings_df = warnings_df.to_html(escape=False, classes='df" id = "warning_df')
errors_df = errors_df.to_html(escape=False, classes='df" id = "error_df')
template = Template(open(os.path.join(sys.path[0], "template.html")).read())
s = template.render(name=metadata['name'], run_id=metadata['run_id'],
level_counts=level_counts,
info_counts=info_counts,
warning_counts=warning_counts,
error_counts=error_counts,
warnings_df=warnings_df, errors_df=errors_df, info_df=info_df)
out_path = log_path.rsplit(".", 1)[0] + ".html"
with open(out_path, 'w') as f:
f.write(s)
if show_browser:
webbrowser.open(out_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('logfiles', type=str, nargs='+')
parser.add_argument('--fields', type=str, default='ret,avglen,ent,kl,vf_r2,ttotal')
parser.add_argument('--noplot', action='store_true')
parser.add_argument('--plotfile', type=str, default=None)
parser.add_argument('--range_end', type=int, default=None)
args = parser.parse_args()
assert len(set(args.logfiles)) == len(args.logfiles), 'Log files must be unique'
fields = args.fields.split(',')
# Load logs from all files
fname2log = {}
for fname in args.logfiles:
if ':' in fname:
os.system('rsync -avrz {} /tmp'.format(fname))
fname = os.path.join('/tmp', os.path.basename(fname))
with pd.HDFStore(fname, 'r') as f:
assert fname not in fname2log
df = f['log']
df.set_index('iter', inplace=True)
fname2log[fname] = df.loc[:args.range_end, fields]
# Print
if not args.noplot or args.plotfile is not None:
import matplotlib
if args.plotfile is not None:
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.style.use('seaborn-colorblind')
ax = None
for fname, df in fname2log.items():
with pd.option_context('display.max_rows', 9999):
print(fname)
print(df[-1:])
if 'vf_r2' in df.keys():
df['vf_r2'] = np.maximum(0, df['vf_r2'])
if not args.noplot:
if ax is None:
ax = df.plot(subplots=True, title=','.join(args.logfiles))
else:
df.plot(subplots=True, title=','.join(args.logfiles), ax=ax, legend=False)
if args.plotfile is not None:
plt.savefig(args.plotfile, transparent=True, bbox_inches='tight', dpi=300)
elif not args.noplot:
plt.show()
def transform(self, X, y=None):
# Suppress SettingWithCopyWarning (alternatively: add a X = X.copy()
with pd.option_context('mode.chained_assignment', None):
# --- Convert Embarked
mapping = {'S': 0,
'C': 1,
'Q': 2,
}
X.loc[:, 'Embarked'] = X.loc[:, 'Embarked'].replace(mapping, inplace=False)
# --- Convert Sex
mapping = {'female': 0,
'male': 1
}
X.loc[:, 'Sex'] = X['Sex'].replace(mapping, inplace=False)
# --- Convert Name to Title
X.loc[:, 'Title'] = X['Name'].map(lambda name: name.split(',')[1].split('.')[0].strip())
# a map of more aggregated titles
mapping = {
"Capt": 0, # Officer
"Col": 0, # Officer
"Major": 0, # Officer
"Jonkheer": 1, # Royalty
"Don": 1, # Royalty
"Sir": 1, # Royalty
"Dr": 0, # Officer
"Rev": 0, # Officer
"the Countess": 1, # Royalty
"Dona": 1, # Royalty
"Mme": 2, # "Mrs"
"Mlle": 3, # "Miss"
"Ms": 2, # "Mrs"
"Mr": 4, # "Mr"
"Mrs": 2, # "Mrs"
"Miss": 3, # "Miss"
"Master": 5, # "Master"
"Lady": 1 # "Royalty"
}
X.loc[:, 'Title'] = X['Title'].map(mapping)
X = X.drop('Name', 1)
return X
def test_representation_to_series(self):
idx1 = DatetimeIndex([], freq='D')
idx2 = DatetimeIndex(['2011-01-01'], freq='D')
idx3 = DatetimeIndex(['2011-01-01', '2011-01-02'], freq='D')
idx4 = DatetimeIndex(
['2011-01-01', '2011-01-02', '2011-01-03'], freq='D')
idx5 = DatetimeIndex(['2011-01-01 09:00', '2011-01-01 10:00',
'2011-01-01 11:00'], freq='H', tz='Asia/Tokyo')
idx6 = DatetimeIndex(['2011-01-01 09:00', '2011-01-01 10:00', pd.NaT],
tz='US/Eastern')
idx7 = DatetimeIndex(['2011-01-01 09:00', '2011-01-02 10:15'])
exp1 = """Series([], dtype: datetime64[ns])"""
exp2 = """0 2011-01-01
dtype: datetime64[ns]"""
exp3 = """0 2011-01-01
1 2011-01-02
dtype: datetime64[ns]"""
exp4 = """0 2011-01-01
1 2011-01-02
2 2011-01-03
dtype: datetime64[ns]"""
exp5 = """0 2011-01-01 09:00:00+09:00
1 2011-01-01 10:00:00+09:00
2 2011-01-01 11:00:00+09:00
dtype: datetime64[ns, Asia/Tokyo]"""
exp6 = """0 2011-01-01 09:00:00-05:00
1 2011-01-01 10:00:00-05:00
2 NaT
dtype: datetime64[ns, US/Eastern]"""
exp7 = """0 2011-01-01 09:00:00
1 2011-01-02 10:15:00
dtype: datetime64[ns]"""
with pd.option_context('display.width', 300):
for idx, expected in zip([idx1, idx2, idx3, idx4,
idx5, idx6, idx7],
[exp1, exp2, exp3, exp4,
exp5, exp6, exp7]):
result = repr(Series(idx))
self.assertEqual(result, expected)
def test_repr_max_columns_max_rows(self):
term_width, term_height = get_terminal_size()
if term_width < 10 or term_height < 10:
raise nose.SkipTest("terminal size too small, "
"{0} x {1}".format(term_width, term_height))
def mkframe(n):
index = ['%05d' % i for i in range(n)]
return DataFrame(0, index, index)
df6 = mkframe(6)
df10 = mkframe(10)
with option_context('mode.sim_interactive', True):
with option_context('display.width', term_width * 2):
with option_context('display.max_rows', 5,
'display.max_columns', 5):
self.assertFalse(has_expanded_repr(mkframe(4)))
self.assertFalse(has_expanded_repr(mkframe(5)))
self.assertFalse(has_expanded_repr(df6))
self.assertTrue(has_doubly_truncated_repr(df6))
with option_context('display.max_rows', 20,
'display.max_columns', 10):
# Out off max_columns boundary, but no extending
# since not exceeding width
self.assertFalse(has_expanded_repr(df6))
self.assertFalse(has_truncated_repr(df6))
with option_context('display.max_rows', 9,
'display.max_columns', 10):
# out vertical bounds can not result in exanded repr
self.assertFalse(has_expanded_repr(df10))
self.assertTrue(has_vertically_truncated_repr(df10))
# width=None in terminal, auto detection
with option_context('display.max_columns', 100, 'display.max_rows',
term_width * 20, 'display.width', None):
df = mkframe((term_width // 7) - 2)
self.assertFalse(has_expanded_repr(df))
df = mkframe((term_width // 7) + 2)
com.pprint_thing(df._repr_fits_horizontal_())
self.assertTrue(has_expanded_repr(df))
def to_clipboard(obj, excel=None, sep=None, **kwargs): # pragma: no cover
"""
Attempt to write text representation of object to the system clipboard
The clipboard can be then pasted into Excel for example.
Parameters
----------
obj : the object to write to the clipboard
excel : boolean, defaults to True
if True, use the provided separator, writing in a csv
format for allowing easy pasting into excel.
if False, write a string representation of the object
to the clipboard
sep : optional, defaults to tab
other keywords are passed to to_csv
Notes
-----
Requirements for your platform
- Linux: xclip, or xsel (with gtk or PyQt4 modules)
- Windows:
- OS X:
"""
from pandas.util.clipboard import clipboard_set
if excel is None:
excel = True
if excel:
try:
if sep is None:
sep = '\t'
buf = StringIO()
obj.to_csv(buf, sep=sep, **kwargs)
clipboard_set(buf.getvalue())
return
except:
pass
if isinstance(obj, DataFrame):
# str(df) has various unhelpful defaults, like truncation
with option_context('display.max_colwidth', 999999):
objstr = obj.to_string(**kwargs)
else:
objstr = str(obj)
clipboard_set(objstr)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('logfiles', type=str, nargs='+')
parser.add_argument('--fields', type=str, default='trueret,avglen,ent,kl,vf_r2,vf_kl,tdvf_r2,rloss,racc')
parser.add_argument('--noplot', action='store_true')
parser.add_argument('--plotfile', type=str, default=None)
parser.add_argument('--range_end', type=int, default=None)
args = parser.parse_args()
assert len(set(args.logfiles)) == len(args.logfiles), 'Log files must be unique'
fields = args.fields.split(',')
# Load logs from all files
fname2log = {}
for fname in args.logfiles:
with pd.HDFStore(fname, 'r') as f:
assert fname not in fname2log
df = f['log']
df.set_index('iter', inplace=True)
fname2log[fname] = df.loc[:args.range_end, fields]
# Print stuff
if not args.noplot or args.plotfile is not None:
import matplotlib
if args.plotfile is not None:
matplotlib.use('Agg')
import matplotlib.pyplot as plt; plt.style.use('ggplot')
ax = None
for fname, df in fname2log.items():
with pd.option_context('display.max_rows', 9999):
print fname
print df[-1:]
df['vf_r2'] = np.maximum(0,df['vf_r2'])
if ax is None:
ax = df.plot(subplots=True, title=fname)
else:
df.plot(subplots=True, title=fname, ax=ax, legend=False)
if not args.noplot:
plt.show()
if args.plotfile is not None:
plt.savefig(args.plotfile, bbox_inches='tight', dpi=200)
def _print_df_scores(df_scores, score_types, indent=''):
"""Pretty print the scores dataframe.
Parameters
----------
df_scores : pd.DataFrame
the score dataframe
score_types : list of score types
a list of score types to use
indent : str, default=''
indentation if needed
"""
try:
# try to re-order columns/rows in the printed array
# we may not have all train, valid, test, so need to select
index_order = np.array(['train', 'valid', 'test'])
ordered_index = index_order[np.isin(index_order, df_scores.index)]
df_scores = df_scores.loc[
ordered_index, [score_type.name for score_type in score_types]]
except Exception:
_print_warning("Couldn't re-order the score matrix..")
with pd.option_context("display.width", 160):
df_repr = repr(df_scores)
df_repr_out = []
for line, color_key in zip(df_repr.splitlines(),
[None, None] +
list(df_scores.index.values)):
if line.strip() == 'step':
continue
if color_key is None:
# table header
line = stylize(line, fg(fg_colors['title']) + attr('bold'))
if color_key is not None:
tokens = line.split()
tokens_bak = tokens[:]
if 'official_' + color_key in fg_colors:
# line label and official score bold & bright
label_color = fg(fg_colors['official_' + color_key])
tokens[0] = stylize(tokens[0], label_color + attr('bold'))
tokens[1] = stylize(tokens[1], label_color + attr('bold'))
if color_key in fg_colors:
# other scores pale
tokens[2:] = [stylize(token, fg(fg_colors[color_key]))
for token in tokens[2:]]
for token_from, token_to in zip(tokens_bak, tokens):
line = line.replace(token_from, token_to)
line = indent + line
df_repr_out.append(line)
print('\n'.join(df_repr_out))