From 3e25b3ae4ab098eccda98467c93c9b1e1554ea45 Mon Sep 17 00:00:00 2001 From: Mostafa Safaie Date: Wed, 23 Apr 2025 16:01:42 +0100 Subject: [PATCH 1/4] add `download_animal` --- bnd/__init__.py | 5 +++++ bnd/data_transfer.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/bnd/__init__.py b/bnd/__init__.py index 9b5d10d..3407ef0 100644 --- a/bnd/__init__.py +++ b/bnd/__init__.py @@ -1,6 +1,11 @@ import logging import warnings +# Package interface +from .cli import dl, up, to_pyal, batch_ks +from .data_transfer import upload_session, download_session, download_animal + +__all__ = [dl, up, to_pyal, batch_ks, upload_session, download_session, download_animal] # Create a logger for the package def set_logging( diff --git a/bnd/data_transfer.py b/bnd/data_transfer.py index 68637ef..9b38411 100644 --- a/bnd/data_transfer.py +++ b/bnd/data_transfer.py @@ -132,3 +132,14 @@ def download_session(session_name: str, file_extension: str, max_size_MB: float, logger.info(f'"{file.name}" is too large. Skipping.') logger.info("Download complete.") + +def download_animal(animal_name: str, file_extension: str, max_size_MB: float, do_video: bool) -> None: + """ + Download a session from the server. + """ + config = _load_config() + + remote_animal_path = config.get_remote_animal_path(animal_name) + _,session_list = list_session_datetime(remote_animal_path) + for session_name in session_list: + download_session(session_name, file_extension, max_size_MB, do_video) From 6f22a1e5cdaface14eda65a7cb076ba140306e7d Mon Sep 17 00:00:00 2001 From: Mostafa Safaie Date: Thu, 24 Apr 2025 15:44:56 +0100 Subject: [PATCH 2/4] make all imports relative --- bnd/__init__.py | 32 ------------------ bnd/cli.py | 8 ++--- bnd/config.py | 2 +- bnd/data_transfer.py | 4 +-- bnd/logger.py | 33 +++++++++++++++++++ bnd/pipeline/__init__.py | 6 ++-- bnd/pipeline/kilosort.py | 4 +-- bnd/pipeline/nwb.py | 12 +++---- bnd/pipeline/nwbtools/beneuro_converter.py | 10 +++--- .../nwbtools/multiprobe_kilosort_interface.py | 2 +- .../nwbtools/pycontrol_data_import.py | 2 +- bnd/pipeline/nwbtools/pycontrol_interface.py | 2 +- bnd/pipeline/pyaldata.py | 6 ++-- bnd/update_bnd.py | 4 +-- 14 files changed, 64 insertions(+), 63 deletions(-) create mode 100644 bnd/logger.py diff --git a/bnd/__init__.py b/bnd/__init__.py index 3407ef0..40482d7 100644 --- a/bnd/__init__.py +++ b/bnd/__init__.py @@ -1,38 +1,6 @@ -import logging -import warnings - # Package interface from .cli import dl, up, to_pyal, batch_ks from .data_transfer import upload_session, download_session, download_animal __all__ = [dl, up, to_pyal, batch_ks, upload_session, download_session, download_animal] -# Create a logger for the package -def set_logging( - file_name: str, -) -> logging.Logger: - """ - Set project-wide logging - - Parameters - ---------- - file_name: str - Name of the module being logged - - Returns - ------- - logger: logging.Logger - logger object - """ - logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") - logging.captureWarnings(True) - - logger = logging.getLogger(file_name) - - def custom_warning_handler(message, category, filename, lineno, file=None, line=None): - logger.warning(f"{category.__name__}: {message}") - - # Set the custom handler - warnings.showwarning = custom_warning_handler - - return logger diff --git a/bnd/cli.py b/bnd/cli.py index d60e052..c5c2e37 100644 --- a/bnd/cli.py +++ b/bnd/cli.py @@ -7,7 +7,7 @@ from rich import print -from bnd.config import ( +from .config import ( _check_is_git_track, _check_root, _check_session_directory, @@ -17,9 +17,9 @@ get_last_session, list_session_datetime, ) -from bnd.data_transfer import download_session, upload_session -from bnd.pipeline import _check_processing_dependencies -from bnd.update_bnd import check_for_updates, update_bnd +from .data_transfer import download_session, upload_session +from .pipeline import _check_processing_dependencies +from .update_bnd import check_for_updates, update_bnd # Create a Typer app app = typer.Typer( diff --git a/bnd/config.py b/bnd/config.py index 9746cf3..7cb6753 100644 --- a/bnd/config.py +++ b/bnd/config.py @@ -2,7 +2,7 @@ from datetime import datetime from pathlib import Path -from bnd import set_logging +from .logger import set_logging logger = set_logging(__name__) diff --git a/bnd/data_transfer.py b/bnd/data_transfer.py index 9b38411..832d4c2 100644 --- a/bnd/data_transfer.py +++ b/bnd/data_transfer.py @@ -3,8 +3,8 @@ import shutil from pathlib import Path -from bnd import set_logging -from bnd.config import _load_config, list_session_datetime +from .logger import set_logging +from .config import _load_config, list_session_datetime logger = set_logging(__name__) diff --git a/bnd/logger.py b/bnd/logger.py new file mode 100644 index 0000000..9b5d10d --- /dev/null +++ b/bnd/logger.py @@ -0,0 +1,33 @@ +import logging +import warnings + + +# Create a logger for the package +def set_logging( + file_name: str, +) -> logging.Logger: + """ + Set project-wide logging + + Parameters + ---------- + file_name: str + Name of the module being logged + + Returns + ------- + logger: logging.Logger + logger object + """ + logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + logging.captureWarnings(True) + + logger = logging.getLogger(file_name) + + def custom_warning_handler(message, category, filename, lineno, file=None, line=None): + logger.warning(f"{category.__name__}: {message}") + + # Set the custom handler + warnings.showwarning = custom_warning_handler + + return logger diff --git a/bnd/pipeline/__init__.py b/bnd/pipeline/__init__.py index aca02c1..fbc79a3 100644 --- a/bnd/pipeline/__init__.py +++ b/bnd/pipeline/__init__.py @@ -5,9 +5,9 @@ def _check_processing_dependencies(): try: - from bnd.pipeline.kilosort import run_kilosort_on_session - from bnd.pipeline.nwb import run_nwb_conversion - from bnd.pipeline.pyaldata import run_pyaldata_conversion + from .kilosort import run_kilosort_on_session + from .nwb import run_nwb_conversion + from .pyaldata import run_pyaldata_conversion except Exception as e: raise ImportError( f"Could not import processing dependencies: {e}. Update your environment " diff --git a/bnd/pipeline/kilosort.py b/bnd/pipeline/kilosort.py index 38aeb58..8b72ced 100644 --- a/bnd/pipeline/kilosort.py +++ b/bnd/pipeline/kilosort.py @@ -6,8 +6,8 @@ from kilosort import run_kilosort from kilosort.utils import PROBE_DIR, download_probes -from bnd import set_logging -from bnd.config import Config, _load_config +from ..logger import set_logging +from ..config import Config, _load_config from ..config import find_file logger = set_logging(__name__) diff --git a/bnd/pipeline/nwb.py b/bnd/pipeline/nwb.py index 34d6a03..0f6f50a 100644 --- a/bnd/pipeline/nwb.py +++ b/bnd/pipeline/nwb.py @@ -4,12 +4,12 @@ import numpy as np from rich import print -from bnd import set_logging -from bnd.config import _load_config -from bnd.pipeline.kilosort import run_kilosort_on_session -from bnd.pipeline.nwbtools.anipose_interface import AniposeInterface -from bnd.pipeline.nwbtools.beneuro_converter import BeNeuroConverter -from bnd.pipeline.nwbtools.multiprobe_kilosort_interface import ( +from ..logger import set_logging +from ..config import _load_config +from .kilosort import run_kilosort_on_session +from .nwbtools.anipose_interface import AniposeInterface +from .nwbtools.beneuro_converter import BeNeuroConverter +from .nwbtools.multiprobe_kilosort_interface import ( MultiProbeKiloSortInterface, ) diff --git a/bnd/pipeline/nwbtools/beneuro_converter.py b/bnd/pipeline/nwbtools/beneuro_converter.py index ad41ecf..c1f4669 100644 --- a/bnd/pipeline/nwbtools/beneuro_converter.py +++ b/bnd/pipeline/nwbtools/beneuro_converter.py @@ -6,13 +6,13 @@ from neuroconv.datainterfaces import SpikeGLXRecordingInterface # PhySortingInterface from neuroconv.tools.signal_processing import get_rising_frames_from_ttl -from bnd import set_logging -from bnd.config import _load_config -from bnd.pipeline.nwbtools.anipose_interface import AniposeInterface -from bnd.pipeline.nwbtools.multiprobe_kilosort_interface import ( +from ...logger import set_logging +from ...config import _load_config +from .anipose_interface import AniposeInterface +from .multiprobe_kilosort_interface import ( MultiProbeKiloSortInterface, ) -from bnd.pipeline.nwbtools.pycontrol_interface import PyControlInterface +from .pycontrol_interface import PyControlInterface logger = set_logging(__name__) config = _load_config() diff --git a/bnd/pipeline/nwbtools/multiprobe_kilosort_interface.py b/bnd/pipeline/nwbtools/multiprobe_kilosort_interface.py index 5a34391..86a9bed 100644 --- a/bnd/pipeline/nwbtools/multiprobe_kilosort_interface.py +++ b/bnd/pipeline/nwbtools/multiprobe_kilosort_interface.py @@ -15,7 +15,7 @@ from neuroconv.utils import DeepDict from pynwb import NWBFile -from bnd import set_logging +from ...logger import set_logging logger = set_logging(__name__) diff --git a/bnd/pipeline/nwbtools/pycontrol_data_import.py b/bnd/pipeline/nwbtools/pycontrol_data_import.py index ff8f2f2..c7b2acd 100644 --- a/bnd/pipeline/nwbtools/pycontrol_data_import.py +++ b/bnd/pipeline/nwbtools/pycontrol_data_import.py @@ -8,7 +8,7 @@ import numpy as np -from bnd import set_logging +from ...logger import set_logging logger = set_logging(__name__) diff --git a/bnd/pipeline/nwbtools/pycontrol_interface.py b/bnd/pipeline/nwbtools/pycontrol_interface.py index 0b195ea..7b2ab31 100644 --- a/bnd/pipeline/nwbtools/pycontrol_interface.py +++ b/bnd/pipeline/nwbtools/pycontrol_interface.py @@ -15,7 +15,7 @@ from pynwb.behavior import BehavioralEvents, Position, SpatialSeries from pynwb.epoch import TimeIntervals -from bnd import set_logging +from ...logger import set_logging from .pycontrol_data_import import Event, Print, Session, State diff --git a/bnd/pipeline/pyaldata.py b/bnd/pipeline/pyaldata.py index 0f192ab..78de4eb 100644 --- a/bnd/pipeline/pyaldata.py +++ b/bnd/pipeline/pyaldata.py @@ -12,9 +12,9 @@ from pynwb.behavior import SpatialSeries from pynwb.misc import Units -from bnd import set_logging -from bnd.config import _load_config -from bnd.pipeline.nwb import run_nwb_conversion +from ..logger import set_logging +from ..config import _load_config +from ..pipeline.nwb import run_nwb_conversion logger = set_logging(__name__) diff --git a/bnd/update_bnd.py b/bnd/update_bnd.py index 259d408..8ceef12 100644 --- a/bnd/update_bnd.py +++ b/bnd/update_bnd.py @@ -3,8 +3,8 @@ import warnings from pathlib import Path -from bnd import set_logging -from bnd.config import _load_config +from .logger import set_logging +from .config import _load_config logger = set_logging(__name__) From 07f3622a8353b9e7948fa85768911805d4df547a Mon Sep 17 00:00:00 2001 From: Mostafa Safaie Date: Thu, 24 Apr 2025 16:26:10 +0100 Subject: [PATCH 3/4] expose `init` --- bnd/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bnd/__init__.py b/bnd/__init__.py index 40482d7..765b276 100644 --- a/bnd/__init__.py +++ b/bnd/__init__.py @@ -1,6 +1,6 @@ # Package interface -from .cli import dl, up, to_pyal, batch_ks +from .cli import dl, up, to_pyal, batch_ks, init from .data_transfer import upload_session, download_session, download_animal -__all__ = [dl, up, to_pyal, batch_ks, upload_session, download_session, download_animal] +__all__ = [dl, up, to_pyal, batch_ks, upload_session, download_session, download_animal, init] From 1596de92f5c95b763c1489100a41feaca94ebb2f Mon Sep 17 00:00:00 2001 From: Mostafa Safaie Date: Fri, 25 Apr 2025 11:32:55 +0100 Subject: [PATCH 4/4] add doctring to `download_animal` --- bnd/data_transfer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bnd/data_transfer.py b/bnd/data_transfer.py index 832d4c2..0c8e72c 100644 --- a/bnd/data_transfer.py +++ b/bnd/data_transfer.py @@ -133,9 +133,17 @@ def download_session(session_name: str, file_extension: str, max_size_MB: float, logger.info("Download complete.") -def download_animal(animal_name: str, file_extension: str, max_size_MB: float, do_video: bool) -> None: +def download_animal(animal_name: str, file_extension: str, max_size_MB: float = 0, do_video: bool = False) -> None: """ - Download a session from the server. + Download a all the data of an animal from the server. + animal_name: str = 'M123' + Name of the animal to download + file_extension: str = '.log' + One file type to download + max_size_MB: float = 0 + Maximum size in MB. Any smaller file will be downloaded. Zero mean infinite size + do_video: bool = False + Download video files as well, if they are smaller than `max_size_MB`. No video files by default. """ config = _load_config()