From 1b6382798caca20dce0f81e8fd11f68df75c5c5e Mon Sep 17 00:00:00 2001 From: Hatem Oraby Date: Fri, 1 Feb 2019 13:08:28 +0100 Subject: [PATCH 01/13] hdf5 recursive group save: Print key name for unsupported types --- caiman/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index 8fb91767f..b8c69e7f0 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -442,7 +442,7 @@ def recursively_save_dict_contents_to_group(h5file, path, dic): elif type(item).__name__ in ['CNMFParams', 'Estimates']: # parameter object recursively_save_dict_contents_to_group(h5file, path + key + '/', item.__dict__) else: - raise ValueError('Cannot save %s type.' % type(item)) + raise ValueError("Cannot save %s type for key '%s'." % (type(item), key)) def recursively_load_dict_contents_from_group( h5file, path): From 03a2f7113b35fd98d10ef72f8b1df544e9ccb9a7 Mon Sep 17 00:00:00 2001 From: Hatem Oraby Date: Fri, 1 Feb 2019 13:08:36 +0100 Subject: [PATCH 02/13] hdf5 recursive group save: Add support for missing np.int32 --- caiman/utils/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index b8c69e7f0..842c6cdc5 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -410,8 +410,8 @@ def recursively_save_dict_contents_to_group(h5file, path, dic): item = np.array(item) if not isinstance(key, str): raise ValueError("dict keys must be strings to save to hdf5") - # save strings, numpy.int64, and numpy.float64 types - if isinstance(item, (np.int64, np.float64, str, np.float, float, np.float32,int)): + # save strings, numpy.int64, numpy.int32, and numpy.float64 types + if isinstance(item, (np.int64, np.int32, np.float64, str, np.float, float, np.float32,int)): h5file[path + key] = item if not h5file[path + key].value == item: raise ValueError('The data representation in the HDF5 file does not match the original dict.') From 6973e00094c33a39b5086d795d30f09e255b5a7f Mon Sep 17 00:00:00 2001 From: Hatem Oraby Date: Fri, 1 Feb 2019 13:54:57 +0100 Subject: [PATCH 03/13] Load AVI: Fix get_file_size() assigning to tuple --- caiman/source_extraction/cnmf/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caiman/source_extraction/cnmf/utilities.py b/caiman/source_extraction/cnmf/utilities.py index bda96d659..3c4a4e2c2 100644 --- a/caiman/source_extraction/cnmf/utilities.py +++ b/caiman/source_extraction/cnmf/utilities.py @@ -911,7 +911,7 @@ def get_file_size(file_name, var_name_hdf5=None): T, dims = siz[0], siz[1:] elif extension == '.avi': cap = cv2.VideoCapture(file_name) - dims = (0, 0) + dims = [0, 0] try: T = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) dims[0] = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) From db80ae55cd007a08b2537b3a3936c2f9f7f2c239 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 1 Feb 2019 11:31:23 -0500 Subject: [PATCH 04/13] Have download_demo() display the URL it's fetching, to make debugging failures easier --- caiman/utils/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index 8fb91767f..a190f828f 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -84,7 +84,8 @@ def download_demo(name='Sue_2x_3000_40_-46.tif', save_folder=''): path_movie = os.path.join(base_folder, save_folder, name) if not os.path.exists(path_movie): url = file_dict[name] - logging.info("downloading " + str(name) + " with urllib") + logging.info(f"downloading {name} with urllib") + logging.info(f"GET {url} HTTP/1.1") f = urlopen(url) data = f.read() with open(path_movie, "wb") as code: From e1a481edad9e469a4bce0a091a07fcff3ad7cbbe Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 11 Feb 2019 11:57:31 -0500 Subject: [PATCH 05/13] mmapping: Make code less likely to overflow with explicit casts, better logging --- caiman/mmapping.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/caiman/mmapping.py b/caiman/mmapping.py index 05a2d0c87..2e6c24883 100644 --- a/caiman/mmapping.py +++ b/caiman/mmapping.py @@ -270,13 +270,14 @@ def save_portion(pars): del big_mov else: with open(big_mov, 'r+b') as f: - f.seek(idx_start * Yr_tot.dtype.itemsize * tot_frames) + f.seek(idx_start * np.uint64(Yr_tot.dtype.itemsize) * tot_frames) f.write(Yr_tot) - if f.tell() != idx_end * Yr_tot.dtype.itemsize * tot_frames: - logging.debug(f.tell()) - logging.debug(idx_end * Yr_tot.dtype.itemsize * tot_frames) + computed_position = idx_end * np.uint64(Yr_tot.dtype.itemsize) * tot_frames + if f.tell() != computed_position: + logging.debug(f"Error in mmap portion write: at position {f.tell()}") + logging.debug(f"But should be at position {idx_end} * {Yr_tot.dtype.itemsize} * {tot_frames} = {computed_position}") f.close() - raise Exception('Writing at the wrong location!') + raise Exception('Internal error in mmapping: Actual position does not match computed position') del Yr_tot logging.debug('done') From d4f118009aaa34b728d9dd2c8815bddda81cbbb7 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 11 Feb 2019 12:01:03 -0500 Subject: [PATCH 06/13] Bump up prio of logs before exception --- caiman/mmapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caiman/mmapping.py b/caiman/mmapping.py index 2e6c24883..476f1a739 100644 --- a/caiman/mmapping.py +++ b/caiman/mmapping.py @@ -274,8 +274,8 @@ def save_portion(pars): f.write(Yr_tot) computed_position = idx_end * np.uint64(Yr_tot.dtype.itemsize) * tot_frames if f.tell() != computed_position: - logging.debug(f"Error in mmap portion write: at position {f.tell()}") - logging.debug(f"But should be at position {idx_end} * {Yr_tot.dtype.itemsize} * {tot_frames} = {computed_position}") + logging.critical(f"Error in mmap portion write: at position {f.tell()}") + logging.critical(f"But should be at position {idx_end} * {Yr_tot.dtype.itemsize} * {tot_frames} = {computed_position}") f.close() raise Exception('Internal error in mmapping: Actual position does not match computed position') From 6495ffc25d13eed664dc743cb69cbbe4d7e919f9 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Mon, 11 Feb 2019 13:25:38 -0500 Subject: [PATCH 07/13] mmapping: adjust cast --- caiman/mmapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caiman/mmapping.py b/caiman/mmapping.py index 476f1a739..751957931 100644 --- a/caiman/mmapping.py +++ b/caiman/mmapping.py @@ -270,9 +270,9 @@ def save_portion(pars): del big_mov else: with open(big_mov, 'r+b') as f: - f.seek(idx_start * np.uint64(Yr_tot.dtype.itemsize) * tot_frames) + f.seek(idx_start * int(Yr_tot.dtype.itemsize) * tot_frames) f.write(Yr_tot) - computed_position = idx_end * np.uint64(Yr_tot.dtype.itemsize) * tot_frames + computed_position = idx_end * int(Yr_tot.dtype.itemsize) * tot_frames if f.tell() != computed_position: logging.critical(f"Error in mmap portion write: at position {f.tell()}") logging.critical(f"But should be at position {idx_end} * {Yr_tot.dtype.itemsize} * {tot_frames} = {computed_position}") From a305cb7d50ff28dc0df91ca4035d176a7cc5c7db Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 12 Feb 2019 14:50:28 -0500 Subject: [PATCH 08/13] Remove conditional import that was for Python 2/3 compatibility --- caiman/utils/utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index ded4751c7..01d4c7e11 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -6,7 +6,7 @@ See Also ------------ -https://docs.python.org/2/library/urllib.html +https://docs.python.org/3/library/urllib.request.htm """ @@ -36,11 +36,7 @@ except: pass -# TODO: Simplify conditional imports below -try: - from urllib2 import urlopen -except ImportError: - from urllib.request import urlopen +from urllib.request import urlopen from ..external.cell_magic_wand import cell_magic_wand from ..source_extraction.cnmf.spatial import threshold_components From 629e2ecc71f5892181893b22beaa64de117a501e Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 12 Feb 2019 14:54:50 -0500 Subject: [PATCH 09/13] Provide a modern Dockerfile --- Dockerfile | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 842c1769d..30fb4f28d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,7 @@ -FROM ubuntu +FROM continuumio/anaconda3 -RUN apt-get update -RUN apt-get install -y libglib2.0-0 -RUN apt-get install -y git wget -RUN apt-get install bzip2 -RUN apt-get install -y gcc -RUN apt-get install -y g++ -RUN apt-get install -y libgtk2.0-0 -RUN apt-get install -y xvfb -RUN export MINICONDA=$HOME/miniconda -RUN export PATH="$MINICONDA/bin:$PATH" -RUN hash -r -RUN wget -q https://3230d63b5fc54e62148e-c95ac804525aac4b6dba79b00b39d1d3.ssl.cf1.rackcdn.com/Anaconda2-2.5.0-Linux-x86_64.sh -O anaconda.sh -# RUN bash anaconda.sh -b -p $ANACONDA -RUN bash anaconda.sh -p /anaconda -b -ENV PATH=/anaconda/bin:${PATH} RUN conda config --set always_yes yes RUN conda update --yes conda -RUN conda info -a -RUN CONDA_SSL_VERIFY=false conda update pyopenssl -# RUN conda install -c menpo opencv3=3.1.0 -# RUN conda install -c cvxgrp cvxpy -# RUN conda install -c https://conda.anaconda.org/conda-forge tifffile -# RUN git clone --recursive -b agiovann-master https://github.com/valentina-s/Constrained_NMF.git -# RUN git clone --recursive https://github.com/agiovann/Constrained_NMF.git -# RUN git clone --recursive -b dev https://github.com/agiovann/Constrained_NMF.git -ADD . /CaImAn -WORKDIR /CaImAn/ -RUN conda env update -f environment.yml -n root -#RUN conda install --file requirements_conda.txt -#RUN pip install -r requirements_pip.txt -RUN apt-get install libc6-i386 -RUN apt-get install -y libsm6 libxrender1 -RUN conda install pyqt=4.11.4 -RUN python setup.py install -RUN python setup.py build_ext -i +RUN mkdir src && cd src && git clone -b dev git@github.com:flatironinstitute/CaImAn.git && cd CaImAn && conda env create -n caiman -f environment.yml && conda activate caiman && pip install . +RUN conda activate caiman && caimanmanager.py install -# RUN nosetests - -EXPOSE 8080 From 1f5b22f0d50f82dfc359ac2aea8278b338536143 Mon Sep 17 00:00:00 2001 From: epnev Date: Tue, 12 Feb 2019 16:30:45 -0500 Subject: [PATCH 10/13] pin ipykernel to resolve spyder slowdown issues --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 6467b33b3..b6c3194f0 100644 --- a/environment.yml +++ b/environment.yml @@ -7,6 +7,7 @@ dependencies: - cython - future - h5py +- ipykernel=4.10 - ipython - ipyparallel - jupyter From b03c77b759bc4caea6b9b08a41664ba67300ebdb Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 12 Feb 2019 17:29:36 -0500 Subject: [PATCH 11/13] Inside docker, must not rely on git credentials. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 30fb4f28d..3a7edcc5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,6 @@ FROM continuumio/anaconda3 RUN conda config --set always_yes yes RUN conda update --yes conda -RUN mkdir src && cd src && git clone -b dev git@github.com:flatironinstitute/CaImAn.git && cd CaImAn && conda env create -n caiman -f environment.yml && conda activate caiman && pip install . +RUN mkdir src && cd src && git clone -b dev https://github.com/flatironinstitute/CaImAn.git && cd CaImAn && conda env create -n caiman -f environment.yml && conda activate caiman && pip install . RUN conda activate caiman && caimanmanager.py install From b10d147d23866ba4c10bd7b3d91760d31566620f Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 12 Feb 2019 17:35:14 -0500 Subject: [PATCH 12/13] Dockerfile: continuumio/anaconda3 is not installed in a modern way; adapt --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3a7edcc5f..1ce0e669d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,6 @@ FROM continuumio/anaconda3 RUN conda config --set always_yes yes RUN conda update --yes conda -RUN mkdir src && cd src && git clone -b dev https://github.com/flatironinstitute/CaImAn.git && cd CaImAn && conda env create -n caiman -f environment.yml && conda activate caiman && pip install . -RUN conda activate caiman && caimanmanager.py install +RUN mkdir src && cd src && git clone -b dev https://github.com/flatironinstitute/CaImAn.git && cd CaImAn && conda env create -n caiman -f environment.yml && source activate caiman && pip install . +RUN source activate caiman && caimanmanager.py install From 0c3b07d694229056ef6f26a70f840821280d1ab9 Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Tue, 12 Feb 2019 18:55:43 -0500 Subject: [PATCH 13/13] This dockerfile finally builds correctly. --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ce0e669d..473576ad4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM continuumio/anaconda3 RUN conda config --set always_yes yes RUN conda update --yes conda -RUN mkdir src && cd src && git clone -b dev https://github.com/flatironinstitute/CaImAn.git && cd CaImAn && conda env create -n caiman -f environment.yml && source activate caiman && pip install . -RUN source activate caiman && caimanmanager.py install +RUN apt-get install -y gcc g++ libgl1 +RUN mkdir src && cd src && git clone -b dev https://github.com/flatironinstitute/CaImAn.git && cd CaImAn && conda env create -n caiman -f environment.yml && conda install --override-channels -c conda-forge -n caiman pip +RUN /bin/bash -c "cd src/CaImAn && source activate caiman && /opt/conda/envs/caiman/bin/pip install ." +RUN /bin/bash -c "source activate caiman && caimanmanager.py install"