Merge pull request #1618 from AllenInstitute/rc/2.1.0

2.1.0 release

Author: pickles-bread-and-butter

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.1.0] = 2020-07-16
+
+### Added
+- Behvaior Ophys NWB File writing capability fixes for updated PyNWB and HDMF versions
+- Added warning if using outdated Visual Coding Neuropixels NWB files
+- Added documentation file for Visual Behavior terms in AllenSDK for quick lookup
+
 ## [2.0.0] = 2020-06-11
 
 ### Added

allensdk/__init__.py

@@ -36,7 +36,8 @@
 import logging
 
 
-__version__ = '2.0.2'
+
+__version__ = '2.1.0'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_ophys_api/behavior_ophys_nwb_api.py

@@ -16,6 +16,7 @@
 
 from allensdk.core.lazy_property import LazyProperty
 from allensdk.brain_observatory.nwb.nwb_api import NwbApi
+from allensdk.brain_observatory.nwb.nwb_utils import set_omitted_stop_time
 from allensdk.brain_observatory.behavior.trials_processing import TRIAL_COLUMN_DESCRIPTION_DICT
 from allensdk.brain_observatory.behavior.schemas import OphysBehaviorMetaDataSchema, OphysBehaviorTaskParametersSchema
 from allensdk.brain_observatory.nwb.metadata import load_LabMetaData_extension
@@ -59,6 +60,9 @@ def save(self, session_object):
             stimulus_index = session_object.stimulus_presentations[session_object.stimulus_presentations['image_set'] == nwb_template.name]
             nwb.add_stimulus_index(nwbfile, stimulus_index, nwb_template)
 
+        # search for omitted rows and add stop_time before writing to NWB file
+        set_omitted_stop_time(stimulus_table=session_object.stimulus_presentations)
+
         # Add stimulus presentations data to NWB in-memory object:
         nwb.add_stimulus_presentations(nwbfile, session_object.stimulus_presentations)
 

allensdk/brain_observatory/behavior/stimulus_processing.py

@@ -30,8 +30,17 @@ def load_pickle(pstream):
     return pickle.load(pstream, encoding="bytes")
 
 
-def get_stimulus_presentations(data, stimulus_timestamps):
-
+def get_stimulus_presentations(data, stimulus_timestamps) -> pd.DataFrame:
+    """
+    This function retrieves the stimulus presentation dataframe and
+    renames the columns, adds a stop_time column, and set's index to
+    stimulus_presentation_id before sorting and returning the dataframe.
+    :param data: stimulus file associated with experiment id
+    :param stimulus_timestamps: timestamps indicating when stimuli switched
+                                during experiment
+    :return: stimulus_table: dataframe containing the stimuli metadata as well
+                             as what stimuli was presented
+    """
     stimulus_table = get_visual_stimuli_df(data, stimulus_timestamps)
     # workaround to rename columns to harmonize with visual coding and rebase timestamps to sync time
     stimulus_table.insert(loc=0, column='flash_number', value=np.arange(0, len(stimulus_table)))
@@ -159,7 +168,22 @@ def unpack_change_log(change):
         to_name=to_name,
     )
 
-def get_visual_stimuli_df(data, time):
+
+def get_visual_stimuli_df(data, time) -> pd.DataFrame:
+    """
+    This function loads the stimuli and the omitted stimuli into a dataframe.
+    These stimuli are loaded from the input data, where the set_log and
+    draw_log contained within are used to calculate the epochs. These epochs
+    are used as start_frame and end_frame and converted to times by input
+    stimulus timestamps. The omitted stimuli do not have a end_frame by design
+    though there duration is always 250ms.
+    :param data: the behavior data file
+    :param time: the stimulus timestamps indicating when each stimuli is
+                 displayed
+    :return: df: a pandas dataframe containing the stimuli and omitted stimuli
+                 that were displayed with their frame, end_frame, start_time,
+                 and duration
+    """
 
     stimuli = data['items']['behavior']['stimuli']
     n_frames = len(time)

allensdk/brain_observatory/ecephys/ecephys_session_api/ecephys_nwb_session_api.py

@@ -1,7 +1,9 @@
 from typing import Dict, Union, List, Optional, Callable
 import re
 import ast
+import warnings
 
+import h5py
 import pandas as pd
 import numpy as np
 import xarray as xr
@@ -11,6 +13,7 @@
 from allensdk.brain_observatory.nwb.nwb_api import NwbApi
 import allensdk.brain_observatory.ecephys.nwb  # noqa Necessary to import pyNWB namespaces
 from allensdk.brain_observatory.ecephys import get_unit_filter_value
+from allensdk.brain_observatory.nwb import check_nwbfile_version
 
 color_triplet_re = re.compile(r"\[(-{0,1}\d*\.\d*,\s*)*(-{0,1}\d*\.\d*)\]")
 
@@ -55,6 +58,18 @@ def __init__(self,
         self.additional_unit_metrics = additional_unit_metrics
         self.external_channel_columns = external_channel_columns
 
+        if hasattr(self, "path") and self.path:
+            check_nwbfile_version(
+                nwbfile_path=self.path,
+                desired_minimum_version="2.2.2",
+                warning_msg=(
+                    f"It looks like the Visual Coding Neuropixels nwbfile "
+                    f"you are trying to access ({self.path})"
+                    f"was created by a previous (and incompatible) version of "
+                    f"AllenSDK and pynwb. You will need to either 1) use "
+                    f"AllenSDK version < 2.0.0 or 2) re-download an updated "
+                    f"version of the nwbfile to access the desired data."))
+
     def test(self):
         """ A minimal test to make sure that this API's NWB file exists and is
         readable. Ecephys NWB files use the required session identifier field

allensdk/brain_observatory/nwb/__init__.py

@@ -1,7 +1,9 @@
 import logging
+import warnings
 from pathlib import Path
-from typing import Iterable
+from typing import Iterable, Tuple
 
+import h5py
 import numpy as np
 import pandas as pd
 import datetime
@@ -10,11 +12,12 @@
 import pynwb
 from pynwb.base import TimeSeries, Images
 from pynwb.behavior import BehavioralEvents
-from pynwb import ProcessingModule
+from pynwb import ProcessingModule, NWBFile
 from pynwb.image import ImageSeries, GrayscaleImage, IndexSeries
 from pynwb.ophys import DfOverF, ImageSegmentation, OpticalChannel, Fluorescence
 
 import allensdk.brain_observatory.roi_masks as roi
+from allensdk.brain_observatory.nwb.nwb_utils import (get_column_name)
 from allensdk.brain_observatory.running_speed import RunningSpeed
 from allensdk.brain_observatory import dict_to_indexed_array
 from allensdk.brain_observatory.behavior.image_api import Image
@@ -25,6 +28,50 @@
 
 log = logging.getLogger("allensdk.brain_observatory.nwb")
 
+CELL_SPECIMEN_COL_DESCRIPTIONS = {
+    'cell_specimen_id': 'Unified id of segmented cell across experiments (after'
+                        ' cell matching)',
+    'height': 'Height of ROI in pixels',
+    'width': 'Width of ROI in pixels',
+    'mask_image_plane': 'Which image plane an ROI resides on. Overlapping ROIs '
+                        'are stored on different mask image planes.',
+    'max_correction_down': 'Max motion correction in down direction in pixels',
+    'max_correction_left': 'Max motion correction in left direction in pixels',
+    'max_correction_up': 'Max motion correction in up direction in pixels',
+    'max_correction_right': 'Max motion correction in right direction in '
+                            'pixels',
+    'valid_roi': 'Indicates if cell classification found the ROI to be a cell '
+                 'or not',
+    'x': 'x position of ROI in Image Plane in pixels (top left corner)',
+    'y': 'y position of ROI in Image Plane in pixels (top left corner)'
+}
+
+def check_nwbfile_version(nwbfile_path: str,
+                          desired_minimum_version: str,
+                          warning_msg: str):
+
+        with h5py.File(nwbfile_path, 'r') as f:
+            # nwb 2.x files store version as an attribute
+            try:
+                nwb_version = str(f.attrs["nwb_version"]).split(".")
+            except KeyError:
+                # nwb 1.x files store version as dataset
+                try:
+                    nwb_version = str(f["nwb_version"][...].astype(str))
+                    # Stored in the form: `NWB-x.y.z`
+                    nwb_version = nwb_version.split("-")[1].split(".")
+                except (KeyError, IndexError):
+                    nwb_version = None
+
+        if nwb_version is None:
+            warnings.warn(f"'{nwbfile_path}' doesn't appear to be a valid "
+                          f"Neurodata Without Borders (*.nwb) format file as "
+                          f"neither a 'nwb_version' field nor dataset could "
+                          f"be found!")
+        else:
+            if tuple(nwb_version) < tuple(desired_minimum_version.split(".")):
+                warnings.warn(warning_msg)
+
 
 def read_eye_dlc_tracking_ellipses(input_path: Path) -> dict:
     """Reads eye tracking ellipse fit data from an h5 file.
@@ -424,10 +471,13 @@ def add_stimulus_presentations(nwbfile, stimulus_table, tag='stimulus_time_inter
     """
     stimulus_table = stimulus_table.copy()
     ts = nwbfile.modules['stimulus'].get_data_interface('timestamps')
-    stimulus_names = stimulus_table['stimulus_name'].unique()
+    possible_names = {'stimulus_name', 'image_name'}
+    stimulus_name_column = get_column_name(stimulus_table.columns,
+                                           possible_names)
+    stimulus_names = stimulus_table[stimulus_name_column].unique()
 
     for stim_name in sorted(stimulus_names):
-        specific_stimulus_table = stimulus_table[stimulus_table['stimulus_name'] == stim_name]
+        specific_stimulus_table = stimulus_table[stimulus_table[stimulus_name_column] == stim_name]
         # Drop columns where all values in column are NaN
         cleaned_table = specific_stimulus_table.dropna(axis=1, how='all')
         # For columns with mixed strings and NaNs, fill NaNs with 'N/A'
@@ -447,6 +497,7 @@ def add_stimulus_presentations(nwbfile, stimulus_table, tag='stimulus_time_inter
 
         for row in cleaned_table.itertuples(index=False):
             row = row._asdict()
+
             presentation_interval.add_interval(**row, tags=tag, timeseries=ts)
 
         nwbfile.add_time_intervals(presentation_interval)
@@ -709,7 +760,28 @@ def add_task_parameters(nwbfile, task_parameters):
     nwbfile.add_lab_meta_data(nwb_task_parameters)
 
 
-def add_cell_specimen_table(nwbfile, cell_specimen_table):
+def add_cell_specimen_table(nwbfile: NWBFile,
+                            cell_specimen_table: pd.DataFrame):
+    """
+    This function takes the cell specimen table and writes the ROIs
+    contained within. It writes these to a new NWB imaging plane
+    based off the previously supplied metadata
+    Parameters
+    ----------
+    nwbfile: NWBFile
+        this is the in memory NWBFile currently being written to which ROI data
+        is added
+    cell_specimen_table: pd.DataFrame
+        this is the DataFrame containing the cells segmented from a ophys
+        experiment, stored in json file and loaded.
+        example: /home/nicholasc/projects/allensdk/allensdk/test/
+                 brain_observatory/behavior/cell_specimen_table_789359614.json
+
+    Returns
+    -------
+    nwbfile: NWBFile
+        The altered in memory NWBFile object that now has a specimen table
+    """
     cell_roi_table = cell_specimen_table.reset_index().set_index('cell_roi_id')
 
     # Device:
@@ -769,12 +841,21 @@ def add_cell_specimen_table(nwbfile, cell_specimen_table):
         description="Segmented rois",
         imaging_plane=imaging_plane)
 
-    for c in [c for c in cell_roi_table.columns if c not in ['id', 'mask_matrix']]:
-        plane_segmentation.add_column(c, c)
-
+    for col_name in cell_roi_table.columns:
+        # the columns 'image_mask', 'pixel_mask', and 'voxel_mask' are already defined
+        # in the nwb.ophys::PlaneSegmentation Object
+        if col_name not in ['id', 'mask_matrix', 'image_mask', 'pixel_mask', 'voxel_mask']:
+            # This builds the columns with name of column and description of column
+            # both equal to the column name in the cell_roi_table
+            plane_segmentation.add_column(col_name,
+                                          CELL_SPECIMEN_COL_DESCRIPTIONS.get(col_name,
+                                                                             "No Description Available"))
+
+    # go through each roi and add it to the plan segmentation object
     for cell_roi_id, row in cell_roi_table.iterrows():
         sub_mask = np.array(row.pop('image_mask'))
-        curr_roi = roi.create_roi_mask(fov_width, fov_height, [(fov_width - 1), 0, (fov_height - 1), 0], roi_mask=sub_mask)
+        curr_roi = roi.create_roi_mask(fov_width, fov_height, [(fov_width - 1), 0, (fov_height - 1), 0],
+                                       roi_mask=sub_mask)
         mask = curr_roi.get_mask_plane()
         csid = row.pop('cell_specimen_id')
         row['cell_specimen_id'] = -1 if csid is None else csid

allensdk/brain_observatory/nwb/nwb_utils.py

@@ -0,0 +1,52 @@
+import pandas as pd
+# All of the omitted stimuli have a duration of 250ms as defined
+# by the Visual Behavior team. For questions about duration contact that
+# team.
+
+
+def get_column_name(table_cols: list,
+                    possible_names: set) -> str:
+    """
+    This function returns a column name, given a table with unknown
+    column names and a set of possible column names which are expected.
+    The table column name returned should be the only name contained in
+    the "expected" possible names.
+    :param table_cols: the table columns to search for the possible name within
+    :param possible_names: the names that could exist within the data columns
+    :return: the first entry of the intersection between the possible names
+             and the names of the columns of the stimulus table
+    """
+
+    column_set = set(table_cols)
+    column_names = list(column_set.intersection(possible_names))
+    if not len(column_names) == 1:
+        raise KeyError("Table expected one name column in intersection, found:"
+                       f" {column_names}")
+    return column_names[0]
+
+
+def set_omitted_stop_time(stimulus_table: pd.DataFrame,
+                          omitted_time_duration: float=0.25) -> None:
+    """
+    This function sets the stop time for a row that of a stimuli table that
+    is a omitted stimuli. A omitted stimuli is a stimuli where a mouse is
+    shown only a grey screen and these last for 250 milliseconds. These do not
+    include a stop_time or end_frame as other stimuli in the stimulus table due
+    to design choices. For these stimuli to be added they must have the
+    stop_time calculated and put into the row as data before writing to NWB.
+    :param stimulus_table: pd.DataFrame that contains the stimuli presented to
+                           an experiment subject
+    :param omitted_time_duration: The duration in seconds of the expected length
+                                  of the omitted stimuli
+    :return:
+          stimulus_table_row: returns the same dictionary as inputted but with
+                              an additional entry for stop_time.
+    """
+    omitted_row_indexs = stimulus_table.index[stimulus_table['omitted']].tolist()
+    for omitted_row_idx in omitted_row_indexs:
+        row = stimulus_table.iloc[omitted_row_idx]
+        start_time = row['start_time']
+        end_time = start_time + omitted_time_duration
+        row['stop_time'] = end_time
+        row['duration'] = omitted_time_duration
+        stimulus_table.iloc[omitted_row_idx] = row

allensdk/test/brain_observatory/behavior/test_behavior_data_lims_api.py

@@ -6,20 +6,28 @@
 import math
 
 from allensdk.internal.api.behavior_data_lims_api import BehaviorDataLimsApi
+from allensdk.core.authentication import DbCredentials
 from allensdk.internal.api.behavior_ophys_api import BehaviorOphysLimsApi
 from allensdk.brain_observatory.running_speed import RunningSpeed
 from allensdk.core.exceptions import DataFrameIndexError
 
+mock_db_credentials = DbCredentials(dbname='mock_db', user='mock_user',
+                                      host='mock_host', port='mock_port',
+                                      password='mock')
+
 
 @pytest.fixture
 def MockBehaviorDataLimsApi():
+
     class MockBehaviorDataLimsApi(BehaviorDataLimsApi):
         """
         Mock class that overrides some functions to provide test data and
         initialize without calls to db.
         """
         def __init__(self):
-            super().__init__(behavior_session_id=8675309)
+            super().__init__(behavior_session_id=8675309,
+                             lims_credentials=mock_db_credentials,
+                             mtrain_credentials=mock_db_credentials)
 
         def _get_ids(self):
             return {}
@@ -73,7 +81,9 @@ class MockApiRunSpeedExpectedError(BehaviorDataLimsApi):
         initialize without calls to db.
         """
         def __init__(self):
-            super().__init__(behavior_session_id=8675309)
+            super().__init__(behavior_session_id=8675309,
+                             mtrain_credentials=mock_db_credentials,
+                             lims_credentials=mock_db_credentials)
 
         def _get_ids(self):
             return {}

allensdk/test/brain_observatory/behavior/test_behavior_ophys_session.py

@@ -41,7 +41,6 @@ def test_session_from_json(tmpdir_factory, session_data, get_expected, get_from_
     compare_fields(expected, obtained)
 
 
-@pytest.mark.xfail
 @pytest.mark.requires_bamboo
 def test_nwb_end_to_end(tmpdir_factory):
     oeid = 789359614