Merge branch 'master' of https://github.com/datajoint/datajoint-python into mp-test

Author: A-Baji

CHANGELOG.md

@@ -4,6 +4,7 @@
 * Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
 * Bugfix - Add missing `jobs` argument for multiprocessing
 * Add - Test for multiprocessing
+* Bugfix - Fix external store key name doesn't allow '-' (#1005) PR #1006
 
 ### 0.13.3 -- Feb 9, 2022
 * Bugfix - Fix error in listing ancestors, descendants with part tables.

LNX-docker-compose.yml

@@ -32,7 +32,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.19
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

README.md

@@ -8,10 +8,8 @@
 # Welcome to DataJoint for Python!
 DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data.
 
-DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
-
-Vathes LLC supports DataJoint for Python as an open-source project and everyone is welcome to contribute.
-Its DataJoint Neuro (https://djneuro.io) business provides support to neuroscience labs for developing and executing custom data pipelines.
+DataJoint was initially developed in 2009 by Dimitri Yatsenko in Andreas Tolias' Lab at Baylor College of Medicine for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers.
+Presently, the primary developer of DataJoint open-source software is the company DataJoint (https://datajoint.com). Related resources are listed at https://datajoint.org
 
 ## Installation
 ```
@@ -22,7 +20,18 @@ If you already have an older version of DataJoint installed using `pip`, upgrade
 ```bash
 pip3 install --upgrade datajoint
 ```
+
+## Documentation and Tutorials
+
+* https://datajoint.org  -- start page
+* https://docs.datajoint.org -- up-to-date documentation
+* https://tutorials.datajoint.io -- step-by-step tutorials
+* https://elements.datajoint.org -- catalog of example pipelines
+* https://codebook.datajoint.io -- interactive online tutorials
+
 ## Python Native Blobs
+<details>
+<summary>Click to expand details</summary>
 
 DataJoint 0.12 adds full support for all native python data types in blobs: tuples, lists, sets, dicts, strings, bytes, `None`, and all their recursive combinations.
 The new blobs are a superset of the old functionality and are fully backward compatible.
@@ -92,16 +101,11 @@ the situation, but generally the following strategies may apply:
 As always, be sure that your data is safely backed up before modifying any
 important DataJoint schema or records.
 
-## Documentation and Tutorials
-A number of labs are currently adopting DataJoint and we are quickly getting the documentation in shape in February 2017.
-
-* https://datajoint.io  -- start page
-* https://docs.datajoint.io -- up-to-date documentation
-* https://tutorials.datajoint.io -- step-by-step tutorials
-* https://catalog.datajoint.io -- catalog of example pipelines
+</details>
 
 ## Running Tests Locally
-
+<details>
+<summary>Click to expand details</summary>
 
 * Create an `.env` with desired development environment values e.g.
 ``` sh
@@ -136,9 +140,9 @@ GID=1000
 * Add entry in `/etc/hosts` for `127.0.0.1 fakeservices.datajoint.io`
 
 
-
-
 ### Launch Jupyter Notebook for Interactive Use
 * Navigate to `localhost:8888`
 * Input Jupyter password
 * Launch a notebook i.e. `New > Python 3`
+
+</details>

datajoint/declare.py

@@ -23,10 +23,10 @@
     BOOL=r'bool(ean)?$',   # aliased to tinyint(1)
     TEMPORAL=r'(date|datetime|time|timestamp|year)(\s*\(.+\))?$',
     INTERNAL_BLOB=r'(tiny|small|medium|long|)blob$',
-    EXTERNAL_BLOB=r'blob@(?P<store>[a-z]\w*)$',
+    EXTERNAL_BLOB=r'blob@(?P<store>[a-z][\-\w]*)$',
     INTERNAL_ATTACH=r'attach$',
-    EXTERNAL_ATTACH=r'attach@(?P<store>[a-z]\w*)$',
-    FILEPATH=r'filepath@(?P<store>[a-z]\w*)$',
+    EXTERNAL_ATTACH=r'attach@(?P<store>[a-z][\-\w]*)$',
+    FILEPATH=r'filepath@(?P<store>[a-z][\-\w]*)$',
     UUID=r'uuid$',
     ADAPTED=r'<.+>$'
 ).items()}

docs-parts/intro/Releases_lang1.rst

@@ -3,6 +3,7 @@
 * Add - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995
 * Bugfix - Add missing `jobs` argument for multiprocessing
 * Add - Test for multiprocessing
+* Bugfix - Fix external store key name doesn't allow '-' (#1005) PR #1006
 
 0.13.3 -- Feb 9, 2022
 ----------------------

local-docker-compose.yml

@@ -34,7 +34,7 @@ services:
       interval: 1s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.19
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

tests/schema_adapted.py

@@ -8,7 +8,7 @@
 from . import PREFIX, CONN_INFO, S3_CONN_INFO
 
 stores_config = {
-    'repo_s3': dict(
+    'repo-s3': dict(
         S3_CONN_INFO,
         protocol='s3',
         location='adapted/repo',
@@ -59,7 +59,7 @@ class LayoutToFilepath(dj.AttributeAdapter):
     An adapted data type that saves a graph layout into fixed filepath
     """
 
-    attribute_type = 'filepath@repo_s3'
+    attribute_type = 'filepath@repo-s3'
 
     @staticmethod
     def get(path):
@@ -68,7 +68,7 @@ def get(path):
 
     @staticmethod
     def put(layout):
-        path = Path(dj.config['stores']['repo_s3']['stage'], 'layout.json')
+        path = Path(dj.config['stores']['repo-s3']['stage'], 'layout.json')
         with open(str(path), "w") as f:
             json.dump(layout, f)
         return path

tests/schema_external.py

@@ -22,7 +22,7 @@
         protocol='file',
         location=tempfile.mkdtemp()),
 
-    'repo_s3': dict(
+    'repo-s3': dict(
         S3_CONN_INFO,
         protocol='s3',
         location='dj/repo',
@@ -130,7 +130,7 @@ class FilepathS3(dj.Manual):
     # table for file management 
     fnum : int 
     ---
-    img : filepath@repo_s3  # managed files 
+    img : filepath@repo-s3  # managed files 
     """
 
 dj.errors._switch_filepath_types(False)

tests/test_filepath.py

@@ -91,7 +91,7 @@ def test_filepath(store="repo"):
 
 def test_filepath_s3():
     """ test file management with s3 """
-    test_filepath(store="repo_s3")
+    test_filepath(store="repo-s3")
 
 
 def test_duplicate_upload(store="repo"):
@@ -107,7 +107,7 @@ def test_duplicate_upload(store="repo"):
 
 
 def test_duplicate_upload_s3():
-    test_duplicate_upload(store="repo_s3")
+    test_duplicate_upload(store="repo-s3")
 
 
 @raises(dj.DataJointError)
@@ -128,7 +128,7 @@ def test_duplicate_error(store="repo"):
 
 
 def test_duplicate_error_s3():
-    test_duplicate_error(store="repo_s3")
+    test_duplicate_error(store="repo-s3")
 
 
 def test_filepath_class(table=Filepath(), store="repo"):
@@ -176,12 +176,12 @@ def test_filepath_class_again():
 
 
 def test_filepath_class_s3():
-    test_filepath_class(FilepathS3(), "repo_s3")
+    test_filepath_class(FilepathS3(), "repo-s3")
 
 
 def test_filepath_class_s3_again():
     """test_filepath_class_s3 again to deal with existing remote files"""
-    test_filepath_class(FilepathS3(), "repo_s3")
+    test_filepath_class(FilepathS3(), "repo-s3")
 
 
 def test_filepath_cleanup(table=Filepath(), store="repo"):
@@ -218,7 +218,7 @@ def test_filepath_cleanup(table=Filepath(), store="repo"):
 
 def test_filepath_cleanup_s3():
     """test deletion of filepath entries from external table """
-    store = "repo_s3"
+    store = "repo-s3"
     test_filepath_cleanup(FilepathS3(), store)