Skip to content

Commit 5565a5a

Browse files
aamsteraamster
authored
Merge pull request #2580 from AllenInstitute/fixes_dataframe_indices
Fixes dataframe indices
2 parents 0223f6d + 528fbed commit 5565a5a

File tree

6 files changed

+154
-120
lines changed

6 files changed

+154
-120
lines changed

‎allensdk/brain_observatory/ecephys/optotagging.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ def __init__(self, table: pd.DataFrame):
1515
# pre-processed optotagging_table may use.
1616
table = \
1717
table.rename(columns={"name": "stimulus_name"})
18-
18+
table.index = table.index.rename('id')
1919
super().__init__(name='optotaggging_table', value=table)
2020

2121
@property

‎allensdk/test/brain_observatory/ecephys/test_ecephys_project_lims_api.py‎

Lines changed: 68 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import os
21
import re
32
from unittest import mock
43

@@ -39,7 +38,7 @@ def __call__(self, query, *args, **kwargs):
3938
"no_pa_check": lambda st: "published_at" not in st
4039
},
4140
pd.DataFrame(
42-
{"something": [12, 14]},
41+
{"something": [12, 14]},
4342
index=pd.Index(name="id", data=[5, 6])
4443
)
4544
],
@@ -48,10 +47,11 @@ def __call__(self, query, *args, **kwargs):
4847
{"session_ids": [1, 2, 3]},
4948
pd.DataFrame({"id": [5, 6], "something": [12, 14]}),
5049
{
51-
"filters_sessions": lambda st: re.compile(r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
50+
"filters_sessions": lambda st: re.compile(
51+
r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
5252
},
5353
pd.DataFrame(
54-
{"something": [12, 14]},
54+
{"something": [12, 14]},
5555
index=pd.Index(name="id", data=[5, 6])
5656
)
5757
],
@@ -60,10 +60,11 @@ def __call__(self, query, *args, **kwargs):
6060
{"unit_ids": [1, 2, 3]},
6161
pd.DataFrame({"id": [5, 6], "something": [12, 14]}),
6262
{
63-
"filters_units": lambda st: re.compile(r".+and eu.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
63+
"filters_units": lambda st: re.compile(
64+
r".+and eu.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
6465
},
6566
pd.DataFrame(
66-
{"something": [12, 14]},
67+
{"something": [12, 14]},
6768
index=pd.Index(name="id", data=[5, 6])
6869
)
6970
],
@@ -72,11 +73,14 @@ def __call__(self, query, *args, **kwargs):
7273
{"channel_ids": [1, 2, 3], "probe_ids": [4, 5, 6]},
7374
pd.DataFrame({"id": [5, 6], "something": [12, 14]}),
7475
{
75-
"filters_channels": lambda st: re.compile(r".+and ec.id in \(1,2,3\).*", re.DOTALL).match(st) is not None,
76-
"filters_probes": lambda st: re.compile(r".+and ep.id in \(4,5,6\).*", re.DOTALL).match(st) is not None
76+
"filters_channels": lambda st: re.compile(
77+
r".+and ec.id in \(1,2,3\).*", re.DOTALL).match(
78+
st) is not None,
79+
"filters_probes": lambda st: re.compile(
80+
r".+and ep.id in \(4,5,6\).*", re.DOTALL).match(st) is not None
7781
},
7882
pd.DataFrame(
79-
{"something": [12, 14]},
83+
{"something": [12, 14]},
8084
index=pd.Index(name="id", data=[5, 6])
8185
)
8286
],
@@ -85,11 +89,15 @@ def __call__(self, query, *args, **kwargs):
8589
{"published_at": "2019-10-22"},
8690
pd.DataFrame({"id": [5, 6], "something": [12, 14]}),
8791
{
88-
"checks_pa_not_null": lambda st: re.compile(r".+and es.published_at is not null.*", re.DOTALL).match(st) is not None,
89-
"checks_pa": lambda st: re.compile(r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(st) is not None
92+
"checks_pa_not_null": lambda st: re.compile(
93+
r".+and es.published_at is not null.*", re.DOTALL).match(
94+
st) is not None,
95+
"checks_pa": lambda st: re.compile(
96+
r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(
97+
st) is not None
9098
},
9199
pd.DataFrame(
92-
{"something": [12, 14]},
100+
{"something": [12, 14]},
93101
index=pd.Index(name="id", data=[5, 6])
94102
)
95103
],
@@ -98,12 +106,17 @@ def __call__(self, query, *args, **kwargs):
98106
{"published_at": "2019-10-22", "session_ids": [1, 2, 3]},
99107
pd.DataFrame({"id": [5, 6], "something": [12, 14]}),
100108
{
101-
"checks_pa_not_null": lambda st: re.compile(r".+and es.published_at is not null.*", re.DOTALL).match(st) is not None,
102-
"checks_pa": lambda st: re.compile(r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(st) is not None,
103-
"filters_sessions": lambda st: re.compile(r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
109+
"checks_pa_not_null": lambda st: re.compile(
110+
r".+and es.published_at is not null.*", re.DOTALL).match(
111+
st) is not None,
112+
"checks_pa": lambda st: re.compile(
113+
r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(
114+
st) is not None,
115+
"filters_sessions": lambda st: re.compile(
116+
r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
104117
},
105118
pd.DataFrame(
106-
{"something": [12, 14]},
119+
{"something": [12, 14]},
107120
index=pd.Index(name="id", data=[5, 6])
108121
)
109122
],
@@ -112,63 +125,81 @@ def __call__(self, query, *args, **kwargs):
112125
{"published_at": "2019-10-22", "session_ids": [1, 2, 3]},
113126
pd.DataFrame({"id": [5, 6], "something": [12, 14]}),
114127
{
115-
"checks_pa_not_null": lambda st: re.compile(r".+and es.published_at is not null.*", re.DOTALL).match(st) is not None,
116-
"checks_pa": lambda st: re.compile(r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(st) is not None,
117-
"filters_sessions": lambda st: re.compile(r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
128+
"checks_pa_not_null": lambda st: re.compile(
129+
r".+and es.published_at is not null.*", re.DOTALL).match(
130+
st) is not None,
131+
"checks_pa": lambda st: re.compile(
132+
r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(
133+
st) is not None,
134+
"filters_sessions": lambda st: re.compile(
135+
r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
118136
},
119137
pd.DataFrame(
120-
{"something": [12, 14]},
138+
{"something": [12, 14]},
121139
index=pd.Index(name="id", data=[5, 6])
122140
)
123141
],
124142
[
125143
"get_sessions",
126144
{"published_at": "2019-10-22", "session_ids": [1, 2, 3]},
127-
pd.DataFrame({"id": [5, 6], "something": [12, 14], "genotype": ["foo", np.nan]}),
145+
pd.DataFrame({"id": [5, 6], "something": [12, 14],
146+
"genotype": ["foo", np.nan]}),
128147
{
129-
"checks_pa_not_null": lambda st: re.compile(r".+and es.published_at is not null.*", re.DOTALL).match(st) is not None,
130-
"checks_pa": lambda st: re.compile(r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(st) is not None,
131-
"filters_sessions": lambda st: re.compile(r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
148+
"checks_pa_not_null": lambda st: re.compile(
149+
r".+and es.published_at is not null.*", re.DOTALL).match(
150+
st) is not None,
151+
"checks_pa": lambda st: re.compile(
152+
r".+and es.published_at <= '2019-10-22'.*", re.DOTALL).match(
153+
st) is not None,
154+
"filters_sessions": lambda st: re.compile(
155+
r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
132156
},
133157
pd.DataFrame(
134-
{"something": [12, 14], "genotype": ["foo", "wt"]},
158+
{"something": [12, 14], "genotype": ["foo", "wt"]},
135159
index=pd.Index(name="id", data=[5, 6])
136160
)
137161
],
138162
[
139163
"get_unit_analysis_metrics",
140164
{"ecephys_session_ids": [1, 2, 3]},
141-
pd.DataFrame({"id": [5, 6], "data": [{"a": 1, "b": 2}, {"a": 3, "b": 4}], "ecephys_unit_id": [10, 11]}),
165+
pd.DataFrame(
166+
{"id": [5, 6], "data": [{"a": 1, "b": 2}, {"a": 3, "b": 4}],
167+
"ecephys_unit_id": [10, 11]}),
142168
{
143-
"filters_sessions": lambda st: re.compile(r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
169+
"filters_sessions": lambda st: re.compile(
170+
r".+and es.id in \(1,2,3\).*", re.DOTALL).match(st) is not None
144171
},
145172
pd.DataFrame(
146-
{"id": [5, 6], "a": [1, 3], "b": [2, 4]},
147-
index=pd.Index(name="iecephys_unit_id", data=[10, 11])
173+
{"id": [5, 6], "a": [1, 3], "b": [2, 4]},
174+
index=pd.Index(name="ecephys_unit_id", data=[10, 11])
148175
)
149176
]
150177
])
151178
def test_pg_query(method_name, kwargs, response, checks, expected):
152-
153179
selector = MockSelector(checks, response)
154180

155-
with mock.patch("allensdk.internal.api.psycopg2_select", new=selector) as ptc:
156-
api = epla.EcephysProjectLimsApi.default(lims_credentials=mock_lims_credentials)
181+
with mock.patch("allensdk.internal.api.psycopg2_select",
182+
new=selector) as ptc:
183+
api = epla.EcephysProjectLimsApi.default(
184+
lims_credentials=mock_lims_credentials)
157185
obtained = getattr(api, method_name)(**kwargs)
158-
pd.testing.assert_frame_equal(expected, obtained, check_like=True, check_dtype=False)
186+
pd.testing.assert_frame_equal(expected, obtained, check_like=True,
187+
check_dtype=False)
159188

160189
any_checks_failed = False
161190
for name, result in ptc.passed.items():
162191
if not result:
163192
print(f"check {name} failed")
164193
any_checks_failed = True
165-
194+
166195
if any_checks_failed:
167196
print(ptc.query)
168197
assert not any_checks_failed
169198

170199

171200
WKF_ID = 12345
201+
202+
172203
class MockPgEngine:
173204

174205
def __init__(self, query_pattern):
@@ -220,8 +251,8 @@ def stream(self, url):
220251
]
221252
])
222253
def test_file_getter(method, kwargs, query_pattern, pg_engine_cls):
223-
224254
api = epla.EcephysProjectLimsApi(
225-
postgres_engine=pg_engine_cls(query_pattern), app_engine=MockHttpEngine()
255+
postgres_engine=pg_engine_cls(query_pattern),
256+
app_engine=MockHttpEngine()
226257
)
227-
getattr(api, method)(**kwargs)
258+
getattr(api, method)(**kwargs)

‎allensdk/test/brain_observatory/ecephys/test_ecephys_session.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ def test_get_stimulus_presentations(valid_stimulus_table_api):
300300
"stimulus_name": ['invalid_presentation',
301301
'invalid_presentation', 'a', 'a_movie'],
302302
"phase": [np.nan, np.nan, 120.0, 180.0]
303-
}, index=pd.Index(name='stimulus_presentations_id', data=[0, 1, 2, 3]))
303+
}, index=pd.Index(name='stimulus_presentation_id', data=[0, 1, 2, 3]))
304304

305305
session = EcephysSession(api=valid_stimulus_table_api)
306306
obtained = session.stimulus_presentations[["start_time",
@@ -323,7 +323,7 @@ def test_get_stimulus_presentations_no_invalid_times(just_stim_table_api):
323323
"stop_time": [1/2, 1, 3/2, 2],
324324
'stimulus_name': ['a', 'a', 'a', 'a_movie'],
325325

326-
}, index=pd.Index(name='stimulus_presentations_id', data=[0, 1, 2, 3]))
326+
}, index=pd.Index(name='stimulus_presentation_id', data=[0, 1, 2, 3]))
327327

328328
session = EcephysSession(api=just_stim_table_api)
329329

@@ -468,8 +468,8 @@ def test_empty_presentationwise_spike_times(spike_times_api):
468468
session.stimulus_presentations.index.values,
469469
session.units.index.values)
470470

471-
assert(isinstance(obtained, pd.DataFrame))
472-
assert(obtained.empty)
471+
assert isinstance(obtained, pd.DataFrame)
472+
assert obtained.empty
473473

474474

475475
def test_conditionwise_spike_statistics(spike_times_api):
@@ -503,11 +503,11 @@ def test_empty_conditionwise_spike_statistics(spike_times_api):
503503
stimulus_presentation_ids=session.stimulus_presentations.index.values,
504504
unit_ids=session.units.index.values
505505
)
506-
assert(len(obtained) == 12)
507-
assert(not np.any(obtained['spike_count'])) # check all spike_counts are 0
508-
assert(not np.any(obtained['spike_mean'])) # spike_means are 0
509-
assert(np.all(np.isnan(obtained['spike_std']))) # std/sem is undefined
510-
assert(np.all(np.isnan(obtained['spike_sem'])))
506+
assert len(obtained) == 12
507+
assert not np.any(obtained['spike_count']) # check all spike_counts are 0
508+
assert not np.any(obtained['spike_mean']) # spike_means are 0
509+
assert np.all(np.isnan(obtained['spike_std'])) # std/sem is undefined
510+
assert np.all(np.isnan(obtained['spike_sem']))
511511

512512

513513
def test_get_stimulus_parameter_values(just_stim_table_api):

‎allensdk/test/brain_observatory/ecephys/test_write_nwb.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,8 @@ def test_add_stimulus_presentations_color(
215215
"level": [10., 9., 8., 7.],
216216
"condition": ["a", "a", "b", "c"],
217217
"stimulus_name": ["w", "x", "y", "z"],
218-
"duration": [0.5, 0.5, 0.5, 0.5]})),
218+
"duration": [0.5, 0.5, 0.5, 0.5]},
219+
index=pd.Index(name="id", data=[0, 1, 2, 3]))),
219220
220221
(pd.DataFrame({"start_time": [0., 1., 2., 3.],
221222
"stop_time": [0.5, 1.5, 2.5, 3.5],
@@ -239,6 +240,7 @@ def test_add_optotagging_table_to_nwbfile(
239240

240241
if expected is None:
241242
expected = opto_table.value
243+
expected.index.name = 'id'
242244

243245
pd.testing.assert_frame_equal(obtained, expected, check_like=True)
244246

@@ -1243,7 +1245,8 @@ def test_add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile,
12431245
"eye_height": [6.] * 5,
12441246
"eye_width": [6.] * 5,
12451247
"eye_phi": [3.] * 5},
1246-
index=[3., 4., 5., 6., 7.]),
1248+
index=pd.Index(name="Time (s)",
1249+
data=[3., 4., 5., 6., 7.])),
12471250
# expected_gaze_data
12481251
pd.DataFrame({"raw_eye_area": [3., 5., 7., 9., 11.],
12491252
"raw_pupil_area": [2., 4., 6., 8., 10.],
@@ -1281,7 +1284,8 @@ def test_add_eye_tracking_rig_geometry_data_to_nwbfile(nwbfile,
12811284
np.nan,
12821285
8.,
12831286
10.]},
1284-
index=[3., 4., 5., 6., 7.])
1287+
index=pd.Index(name="Time (s)",
1288+
data=[3., 4., 5., 6., 7.]))
12851289
),
12861290
])
12871291
def test_add_eye_tracking_data_to_nwbfile(nwbfile,

0 commit comments

Comments
 (0)