From 6b9297186ee05ee19eba823b06d19fcd43fd97a3 Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Mon, 8 Oct 2018 15:45:46 -0700 Subject: [PATCH 1/5] Upgrade to Django 1.11 and Python 3.6 --- oh_data_source/nightscout_data.py | 32 +++++++++++++++++-------------- oh_data_source/tasks.py | 5 +++-- oh_data_source/views.py | 2 +- requirements.txt | 6 ++---- runtime.txt | 2 +- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/oh_data_source/nightscout_data.py b/oh_data_source/nightscout_data.py index d3384b0..a7a15da 100644 --- a/oh_data_source/nightscout_data.py +++ b/oh_data_source/nightscout_data.py @@ -6,7 +6,7 @@ import os import random import string -from urlparse import urlparse +from urllib.parse import urlparse import arrow import requests @@ -75,7 +75,7 @@ def get_ns_entries(oh_member, ns_url, file_obj, before_date, after_date): ns_entries_url = ns_url + '/api/v1/entries.json' # Start a JSON array. - file_obj.write('[') + file_obj.write('['.encode()) initial_entry_done = False # Entries after initial are preceded by commas. # Get 8 million second chunks (~ 1/4th year) until none, or start reached. @@ -111,10 +111,11 @@ def get_ns_entries(oh_member, ns_url, file_obj, before_date, after_date): empty_run = 0 for entry in entries_req.json(): if initial_entry_done: - file_obj.write(',') # JSON array separator + file_obj.write(','.encode()) # JSON array separator else: initial_entry_done = True - json.dump(entry, file_obj) + jsonstr = json.dumps(entry) + file_obj.write(jsonstr.encode()) logger.debug('Wrote {} entries to file...'.format(len(entries_req.json()))) else: empty_run += 1 @@ -124,7 +125,7 @@ def get_ns_entries(oh_member, ns_url, file_obj, before_date, after_date): curr_end = curr_start curr_start = curr_end - 5000000000 - file_obj.write(']') # End of JSON array. + file_obj.write(']'.encode()) # End of JSON array. logger.debug('Done writing entries to file.') @@ -146,7 +147,7 @@ def get_ns_devicestatus(oh_member, ns_url, file_obj, before_date, after_date): ns_entries_url = ns_url + '/api/v1/devicestatus.json' # Start a JSON array. - file_obj.write('[') + file_obj.write('['.encode()) initial_entry_done = False # Entries after initial are preceded by commas. # Get 8 million second chunks (~ 1/4th year) until none, or start reached. @@ -183,10 +184,11 @@ def get_ns_devicestatus(oh_member, ns_url, file_obj, before_date, after_date): for item in devicestatus_req.json(): sub_sensitive(item, subs, 'device') if initial_entry_done: - file_obj.write(',') # JSON array separator + file_obj.write(','.encode()) # JSON array separator else: initial_entry_done = True - json.dump(item, file_obj) + jsonstr = json.dumps(item) + file_obj.write(jsonstr.encode()) logger.debug('Wrote {} devicestatus items to file...'.format( len(devicestatus_req.json()))) else: @@ -198,7 +200,7 @@ def get_ns_devicestatus(oh_member, ns_url, file_obj, before_date, after_date): curr_end = curr_start curr_start = curr_end - datetime.timedelta(days=2) - file_obj.write(']') + file_obj.write(']'.encode()) logger.debug('Done writing devicestatus items to file.') @@ -220,7 +222,7 @@ def get_ns_treatments(oh_member, ns_url, file_obj, before_date, after_date): ns_entries_url = ns_url + '/api/v1/treatments.json' # Start a JSON array. - file_obj.write('[') + file_obj.write('['.encode()) initial_entry_done = False # Entries after initial are preceded by commas. # Get 8 million second chunks (~ 1/4th year) until none, or start reached. @@ -257,10 +259,11 @@ def get_ns_treatments(oh_member, ns_url, file_obj, before_date, after_date): for item in treatments_req.json(): sub_sensitive(item, subs, 'enteredBy') if initial_entry_done: - file_obj.write(',') # JSON array separator + file_obj.write(','.encode()) # JSON array separator else: initial_entry_done = True - json.dump(item, file_obj) + jsonstr = json.dumps(item) + file_obj.write(jsonstr.encode()) logger.debug('Wrote {} treatments items to file...'.format( len(treatments_req.json()))) else: @@ -272,7 +275,7 @@ def get_ns_treatments(oh_member, ns_url, file_obj, before_date, after_date): curr_end = curr_start curr_start = curr_end - datetime.timedelta(days=20) - file_obj.write(']') + file_obj.write(']'.encode()) logger.debug('Done writing treatments items to file.') @@ -301,7 +304,8 @@ def ns_data_file(oh_member, data_type, tempdir, ns_url, ns_params = {'count': 1000000} data_req = requests.get(ns_data_url, params=ns_params) if data_req.json(): - json.dump(data_req.json(), file_obj) + jsonstr = json.dumps(data_req.json()) + file_obj.write(jsonstr.encode()) elif data_type == 'treatments': oh_member.last_xfer_status = 'Retrieving treatments data... ({})'.format( arrow.get().format()) diff --git a/oh_data_source/tasks.py b/oh_data_source/tasks.py index 87c3cf0..efb05b7 100644 --- a/oh_data_source/tasks.py +++ b/oh_data_source/tasks.py @@ -13,7 +13,7 @@ import shutil import tempfile import textwrap -from urllib2 import HTTPError +from urllib.error import HTTPError import arrow from celery import shared_task @@ -54,7 +54,8 @@ def xfer_to_open_humans(oh_id, ns_before, ns_after, ns_url, num_submit=0): oh_member, ns_before, ns_after, ns_url, tempdir) oh_member.last_xfer_status = 'Complete' oh_member.save() - except: + except Exception as error: + logger.debug('Exception\n{}'.format(error)) oh_member.last_xfer_status = 'Failed' oh_member.save() finally: diff --git a/oh_data_source/views.py b/oh_data_source/views.py index fceb44d..b799d63 100644 --- a/oh_data_source/views.py +++ b/oh_data_source/views.py @@ -47,7 +47,7 @@ def oh_code_to_member(code): if settings.OH_CLIENT_SECRET and settings.OH_CLIENT_ID and code: data = { 'grant_type': 'authorization_code', - 'redirect_uri': '{}/complete/'.format(APP_BASE_URL), + 'redirect_uri': '{}/complete'.format(APP_BASE_URL), 'code': code, } req = requests.post( diff --git a/requirements.txt b/requirements.txt index 1d41d94..49dee02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,16 @@ -Django==1.10.5 amqp==2.1.4 -argparse==1.2.1 arrow==0.10.0 billiard==3.5.0.2 celery==4.0.2 dj-database-url==0.4.2 +Django==1.11.16 gunicorn==19.6.0 kombu==4.0.2 postgres==2.2.1 -psycopg2==2.6.2 +psycopg2==2.7 python-dateutil==2.6.0 pytz==2016.10 requests==2.12.4 six==1.10.0 vine==1.1.3 whitenoise==3.2.3 -wsgiref==0.1.2 diff --git a/runtime.txt b/runtime.txt index ba85ab9..1935e97 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-2.7.13 \ No newline at end of file +python-3.6.6 From 16b8689b97ca2a0f1894495eab16f0cefa94d4aa Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Tue, 9 Oct 2018 09:44:24 -0700 Subject: [PATCH 2/5] Upgrade to Django 2.1.2 --- oh_data_source/models.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oh_data_source/models.py b/oh_data_source/models.py index 44f6ea3..7fc0ea2 100644 --- a/oh_data_source/models.py +++ b/oh_data_source/models.py @@ -37,7 +37,7 @@ class OpenHumansMember(models.Model): A User account is created for this Open Humans member. """ - user = models.OneToOneField(User) + user = models.OneToOneField(User, on_delete=models.CASCADE) oh_id = models.CharField(max_length=16, primary_key=True, unique=True) access_token = models.CharField(max_length=256) refresh_token = models.CharField(max_length=256) diff --git a/requirements.txt b/requirements.txt index 49dee02..d4a40d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ arrow==0.10.0 billiard==3.5.0.2 celery==4.0.2 dj-database-url==0.4.2 -Django==1.11.16 +Django==2.1.2 gunicorn==19.6.0 kombu==4.0.2 postgres==2.2.1 From 5c0979f6ca92b68ae085a4b2ad8a7d0ae5c66fe5 Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Tue, 9 Oct 2018 14:58:29 -0700 Subject: [PATCH 3/5] Bump psycopg2 version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4a40d6..7185af3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ Django==2.1.2 gunicorn==19.6.0 kombu==4.0.2 postgres==2.2.1 -psycopg2==2.7 +psycopg2==2.7.5 python-dateutil==2.6.0 pytz==2016.10 requests==2.12.4 From a10916e70cdc33b708e1f87c98852bddfa544a3c Mon Sep 17 00:00:00 2001 From: Mad Price Ball Date: Wed, 10 Oct 2018 13:53:58 -0700 Subject: [PATCH 4/5] Updates for heroku deployment --- Pipfile | 17 ++++ Pipfile.lock | 193 +++++++++++++++++++++++++++++++++++++ README.md | 85 +++++++--------- env.example | 9 +- oh_data_source/settings.py | 22 +---- oh_data_source/wsgi.py | 11 +-- requirements.txt | 16 --- 7 files changed, 257 insertions(+), 96 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock delete mode 100644 requirements.txt diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..cac994a --- /dev/null +++ b/Pipfile @@ -0,0 +1,17 @@ +[[source]] +url = "https://pypi.org/simple" +name = "pypi" +verify_ssl = true + +[dev-packages] + +[requires] +python_version = "3.6" + +[packages] +django = "*" +django-heroku = "*" +celery = "*" +arrow = "*" +requests = "*" +gunicorn = "*" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..817abf8 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,193 @@ +{ + "_meta": { + "hash": { + "sha256": "cd574af29a78dbd4d8772ce0743586b7c70164ba0d0f8b295ebffc3ac0190fc9" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.6" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "amqp": { + "hashes": [ + "sha256:073dd02fdd73041bffc913b767866015147b61f2a9bc104daef172fc1a0066eb", + "sha256:eed41946890cd43e8dee44a316b85cf6fee5a1a34bb4a562b660a358eb529e1b" + ], + "version": "==2.3.2" + }, + "arrow": { + "hashes": [ + "sha256:a558d3b7b6ce7ffc74206a86c147052de23d3d4ef0e17c210dd478c53575c4cd" + ], + "index": "pypi", + "version": "==0.12.1" + }, + "billiard": { + "hashes": [ + "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84" + ], + "version": "==3.5.0.4" + }, + "celery": { + "hashes": [ + "sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678", + "sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13" + ], + "index": "pypi", + "version": "==4.2.1" + }, + "certifi": { + "hashes": [ + "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", + "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" + ], + "version": "==2018.8.24" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "dj-database-url": { + "hashes": [ + "sha256:4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163", + "sha256:851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9" + ], + "version": "==0.5.0" + }, + "django": { + "hashes": [ + "sha256:acdcc1f61fdb0a0c82a1d3bf1879a414e7732ea894a7632af7f6d66ec7ab5bb3", + "sha256:efbcad7ebb47daafbcead109b38a5bd519a3c3cd92c6ed0f691ff97fcdd16b45" + ], + "index": "pypi", + "version": "==2.1.2" + }, + "django-heroku": { + "hashes": [ + "sha256:2bc690aab89eedbe01311752320a9a12e7548e3b0ed102681acc5736a41a4762", + "sha256:6af4bc3ae4a9b55eaad6dbe5164918982d2762661aebc9f83d9fa49f6009514e" + ], + "index": "pypi", + "version": "==0.3.1" + }, + "gunicorn": { + "hashes": [ + "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", + "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" + ], + "index": "pypi", + "version": "==19.9.0" + }, + "idna": { + "hashes": [ + "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", + "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + ], + "version": "==2.7" + }, + "kombu": { + "hashes": [ + "sha256:86adec6c60f63124e2082ea8481bbe4ebe04fde8ebed32c177c7f0cd2c1c9082", + "sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af" + ], + "version": "==4.2.1" + }, + "psycopg2": { + "hashes": [ + "sha256:0b9e48a1c1505699a64ac58815ca99104aacace8321e455072cee4f7fe7b2698", + "sha256:0f4c784e1b5a320efb434c66a50b8dd7e30a7dc047e8f45c0a8d2694bfe72781", + "sha256:0fdbaa32c9eb09ef09d425dc154628fca6fa69d2f7c1a33f889abb7e0efb3909", + "sha256:11fbf688d5c953c0a5ba625cc42dea9aeb2321942c7c5ed9341a68f865dc8cb1", + "sha256:19eaac4eb25ab078bd0f28304a0cb08702d120caadfe76bb1e6846ed1f68635e", + "sha256:3232ec1a3bf4dba97fbf9b03ce12e4b6c1d01ea3c85773903a67ced725728232", + "sha256:36f8f9c216fcca048006f6dd60e4d3e6f406afde26cfb99e063f137070139eaf", + "sha256:59c1a0e4f9abe970062ed35d0720935197800a7ef7a62b3a9e3a70588d9ca40b", + "sha256:6506c5ff88750948c28d41852c09c5d2a49f51f28c6d90cbf1b6808e18c64e88", + "sha256:6bc3e68ee16f571681b8c0b6d5c0a77bef3c589012352b3f0cf5520e674e9d01", + "sha256:6dbbd7aabbc861eec6b910522534894d9dbb507d5819bc982032c3ea2e974f51", + "sha256:6e737915de826650d1a5f7ff4ac6cf888a26f021a647390ca7bafdba0e85462b", + "sha256:6ed9b2cfe85abc720e8943c1808eeffd41daa73e18b7c1e1a228b0b91f768ccc", + "sha256:711ec617ba453fdfc66616db2520db3a6d9a891e3bf62ef9aba4c95bb4e61230", + "sha256:844dacdf7530c5c612718cf12bc001f59b2d9329d35b495f1ff25045161aa6af", + "sha256:86b52e146da13c896e50c5a3341a9448151f1092b1a4153e425d1e8b62fec508", + "sha256:985c06c2a0f227131733ae58d6a541a5bc8b665e7305494782bebdb74202b793", + "sha256:a86dfe45f4f9c55b1a2312ff20a59b30da8d39c0e8821d00018372a2a177098f", + "sha256:aa3cd07f7f7e3183b63d48300666f920828a9dbd7d7ec53d450df2c4953687a9", + "sha256:b1964ed645ef8317806d615d9ff006c0dadc09dfc54b99ae67f9ba7a1ec9d5d2", + "sha256:b2abbff9e4141484bb89b96eb8eae186d77bc6d5ffbec6b01783ee5c3c467351", + "sha256:cc33c3a90492e21713260095f02b12bee02b8d1f2c03a221d763ce04fa90e2e9", + "sha256:d7de3bf0986d777807611c36e809b77a13bf1888f5c8db0ebf24b47a52d10726", + "sha256:db5e3c52576cc5b93a959a03ccc3b02cb8f0af1fbbdc80645f7a215f0b864f3a", + "sha256:e168aa795ffbb11379c942cf95bf813c7db9aa55538eb61de8c6815e092416f5", + "sha256:e9ca911f8e2d3117e5241d5fa9aaa991cb22fb0792627eeada47425d706b5ec8", + "sha256:eccf962d41ca46e6326b97c8fe0a6687b58dfc1a5f6540ed071ff1474cea749e", + "sha256:efa19deae6b9e504a74347fe5e25c2cb9343766c489c2ae921b05f37338b18d1", + "sha256:f4b0460a21f784abe17b496f66e74157a6c36116fa86da8bf6aa028b9e8ad5fe", + "sha256:f93d508ca64d924d478fb11e272e09524698f0c581d9032e68958cfbdd41faef" + ], + "version": "==2.7.5" + }, + "python-dateutil": { + "hashes": [ + "sha256:1adb80e7a782c12e52ef9a8182bebeb73f1d7e24e374397af06fb4956c8dc5c0", + "sha256:e27001de32f627c22380a688bcc43ce83504a7bc5da472209b4c70f02829f0b8" + ], + "version": "==2.7.3" + }, + "pytz": { + "hashes": [ + "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", + "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" + ], + "version": "==2018.5" + }, + "requests": { + "hashes": [ + "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", + "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + ], + "index": "pypi", + "version": "==2.19.1" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + }, + "urllib3": { + "hashes": [ + "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", + "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + ], + "version": "==1.23" + }, + "vine": { + "hashes": [ + "sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72", + "sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b" + ], + "version": "==1.1.4" + }, + "whitenoise": { + "hashes": [ + "sha256:133a92ff0ab8fb9509f77d4f7d0de493eca19c6fea973f4195d4184f888f2e02", + "sha256:32b57d193478908a48acb66bf73e7a3c18679263e3e64bfebcfac1144a430039" + ], + "version": "==4.1" + } + }, + "develop": {} +} diff --git a/README.md b/README.md index 12636f2..3fe9068 100644 --- a/README.md +++ b/README.md @@ -8,79 +8,62 @@ https://dataxfer-ns-oh.herokuapp.com/ ### Local Setup and Development -To develop this tool, you can run this app locally by following the instructions below or by using the Heroku-CLI. - -#### 1. Creating a project on Open Humans +#### Creating a project on Open Humans First you will need an Open Humans project to test and develop on. You can set up a project here: https://www.openhumans.org/direct-sharing/projects/oauth2/create/ -The `Enrollment URL` will be a link to your final app page (http://yourapp.herokuapp.com) once development is complete. +The `Enrollment URL` would be a link to a final app page (http://yourapp.herokuapp.com) once development is complete. -Set the `Redirect URL` to exactly: http://127.0.0.1:5000/complete/ +Set the `Redirect URL` to exactly: `http://127.0.0.1:5000/complete` Once created, you can go to https://www.openhumans.org/direct-sharing/projects/manage/ and click on the project name. You will find your `client_id` and`client_secret` here. For more information on OAuth2 setup, go to: https://www.openhumans.org/direct-sharing/oauth2-setup/ -#### 2. Install Dependencies - -A number of software depenencies are required to develop locally with foreman. You can install them using the following commands: - -```sudo apt-get install rabbitmq-server -sudo apt-get install python-dev -sudo apt-get install libpq-dev -sudo gem install foreman -sudo apt-get install virtualenv -``` - -#### 3. Configure Local Environment Variables - -Download the repository and then copy the enviornment variable file using `cp env.example .env` - -Open the `.env` file and replace `CLIENT_ID` and `CLIENT_SECRET` with your project's matching ID and SECRET. - -#### 4. Setup Local Virtual Environment - -Use `virtualenv venv` to create a new virtual environment file. +#### Installation and configuration -Enter the virtual environment with `source venv/bin/activate`. +These instructions expect you to be using the following. Make sure it's set +up for your system first! -Install the requirements using `pip install -r requirements.txt`. +* `pipenv` for local Python development +* the Heroku CLI: https://devcenter.heroku.com/articles/heroku-cli -Finally run `python manage.py migrate`. - +Local setup: -#### 5. Start Local Server +1. Download the git repository, e.g. using `git clone`. +2. Navigate to be inside the repository base directory. +3. Start a local pipenv shell with `pipenv shell` +4. Install dependencies with `pipenv install` +5. Copy the environment variable file using `cp env.example .env` +6. Open the `.env` file and edit to replace `CLIENT_ID` and `CLIENT_SECRET` with your project's matching ID and SECRET (see above) +7. Initialize the database locally with `heroku local:run python manage.py migrate` -Startup your local server in the virtual environment by typing `foreman start`. +#### Running the app locally -Your local site can be loaded by opening a web browser and visiting http://127.0.0.1:5000/ +Inside the repository base, run: `heroku local`. -`Ctrl-C` to quit foreman and `deactivate` to exit the virtual environment. +The app should be available in a local web browser at: http://127.0.0.1:5000 ### Deployment to Heroku Create a new app in Heroku, and link it to your own repository or use the Heroku-CLI to upload files to the heroku server. -Once you have your heroku app setup, go to https://www.openhumans.org/direct-sharing/projects/manage/ and click on the `edit` button next to your project name. Set the `Redirect URL` to exactly: https://your-app-name.herokuapp.com/complete/ - -Next you will need to add the following local enviornment variables in the Config Vars section of your heroku app: - -`APP_BASE_URL`: Set this to exactly https://your-app-name.herokuapp.com - -`OH_ACTIVITY_PAGE`: https://www.openhumans.org/activity/your-project-name - -`OH_CLIENT_ID`: Your Client ID - -`OH_CLIENT_SECRET`: Your Client Secret - -`HEROKU_APP`: true - -`SECRET_KEY` : your_secret_key_here (see http://www.howtogeek.com/howto/30184/10-ways-to-generate-a-random-password-from-the-command-line/) +#### Config in Open Humans -`PYTHONUNBUFFERED` : true +Once you have your heroku app created, go to https://www.openhumans.org/direct-sharing/projects/manage/ and click on the `edit` button next to your project name. Set the `Redirect URL` to exactly: `https://your-app-name.herokuapp.com/complete` -On the Resources tab for your app, edit the Celery Worker to be active. After this, add a CloudAMQP add-on and use the "Little Lemur" version. +#### Config in Heroku +1. You should add the following local environment variables in the Config Vars section of your heroku app: + * `APP_BASE_URL`: Set this to exactly https://your-app-name.herokuapp.com + * `OH_ACTIVITY_PAGE` (optional): https://www.openhumans.org/activity/your-project-name + * `OH_CLIENT_ID`: Your Client ID + * `OH_CLIENT_SECRET`: Your Client Secret + * `SECRET_KEY` : Something long and random + * `PYTHONUNBUFFERED` : true +2. On the Resources tab for your app, edit the Celery Worker to be active. +3. On the same Resources tab, add a CloudAMQP add-on and use the "Little Lemur" version. +4. Deploy your git repository to git (e.g. `git push heroku master`) +5. Initialize your database with `heroku run 'python manage.py migrate'` -Setup is now complete. +Setup is now complete and the Heroku app should hopefully work! diff --git a/env.example b/env.example index f533b34..3010410 100644 --- a/env.example +++ b/env.example @@ -3,13 +3,16 @@ # When this webapp is run with foreman, variables defined in `.env` are # loaded as environment variables. +# Optional, but you probably want this for logging and debugging. +PYTHONUNBUFFERED='true' + +# DEBUG is off unless this is set to true. +DEBUG='True' + # Django secret. Long and random, e.g.: # http://www.howtogeek.com/howto/30184/10-ways-to-generate-a-random-password-from-the-command-line/ SECRET_KEY='secret_key_here' -# DEBUG is True by default. Set to 'False' to turn it off (e.g. in production). -# DEBUG='False' - # Open Humans OAuth2 project settings. OH_ACTIVITY_PAGE='https://www.openhumans.org/activity/your-project-name-should-be-here/' OH_CLIENT_ID='client_id_here' diff --git a/oh_data_source/settings.py b/oh_data_source/settings.py index af53080..7095a0b 100644 --- a/oh_data_source/settings.py +++ b/oh_data_source/settings.py @@ -3,25 +3,17 @@ """ import os -import dj_database_url +import django_heroku # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.getenv('SECRET_KEY', 'sEkritt-kEE_deFawlT') +SECRET_KEY = os.getenv('SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True # False if os.getenv('DEBUG', '').lower() == 'false' else True - -HEROKU_APP = True if os.getenv('HEROKU_APP', '').lower() == 'true' else False - -# Allow all host headers if this is running as a Heroku app. -if HEROKU_APP: - ALLOWED_HOSTS = ['*'] -else: - ALLOWED_HOSTS = [] +DEBUG = True if os.getenv('DEBUG', '').lower() == 'true' else False # Open Humans configuration OH_CLIENT_ID = os.getenv('OH_CLIENT_ID') @@ -82,8 +74,6 @@ 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), } } -db_from_env = dj_database_url.config(conn_max_age=500) -DATABASES['default'].update(db_from_env) # Password validation @@ -146,7 +136,5 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') STATIC_URL = '/static/' -# Simplified static file serving. -# https://warehouse.python.org/project/whitenoise/ -# See also https://devcenter.heroku.com/articles/django-assets -STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage' +# Configure Django App for Heroku. +django_heroku.settings(locals()) diff --git a/oh_data_source/wsgi.py b/oh_data_source/wsgi.py index 2c49299..909715a 100644 --- a/oh_data_source/wsgi.py +++ b/oh_data_source/wsgi.py @@ -1,18 +1,11 @@ """ -WSGI config for oh_data_source_template project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ +WSGI configuration. """ import os from django.core.wsgi import get_wsgi_application -from whitenoise.django import DjangoWhiteNoise -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "oh_data_source.settings") +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oh_data_source.settings') application = get_wsgi_application() -application = DjangoWhiteNoise(application) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 7185af3..0000000 --- a/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -amqp==2.1.4 -arrow==0.10.0 -billiard==3.5.0.2 -celery==4.0.2 -dj-database-url==0.4.2 -Django==2.1.2 -gunicorn==19.6.0 -kombu==4.0.2 -postgres==2.2.1 -psycopg2==2.7.5 -python-dateutil==2.6.0 -pytz==2016.10 -requests==2.12.4 -six==1.10.0 -vine==1.1.3 -whitenoise==3.2.3 From a677b3075a1906c5f2bf28a4969c0004198ba4fa Mon Sep 17 00:00:00 2001 From: Dana Lewis Date: Wed, 10 Oct 2018 19:59:26 -0700 Subject: [PATCH 5/5] Update setup instructions for dev environment * Editing readme for git tracking * Update README with latest instructions and correct order --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3fe9068..6c45762 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ https://dataxfer-ns-oh.herokuapp.com/ First you will need an Open Humans project to test and develop on. You can set up a project here: https://www.openhumans.org/direct-sharing/projects/oauth2/create/ -The `Enrollment URL` would be a link to a final app page (http://yourapp.herokuapp.com) once development is complete. +The `Enrollment URL` would be a link to a final app page (http://yourapp.herokuapp.com) once development is complete. You can pick or use a dummy name in the meantime. Set the `Redirect URL` to exactly: `http://127.0.0.1:5000/complete` @@ -25,28 +25,29 @@ For more information on OAuth2 setup, go to: https://www.openhumans.org/direct-s These instructions expect you to be using the following. Make sure it's set up for your system first! -* `pipenv` for local Python development -* the Heroku CLI: https://devcenter.heroku.com/articles/heroku-cli +* `pipenv` for local Python development (`brew install pipenv` on Mac) +* the Heroku CLI: https://devcenter.heroku.com/articles/heroku-cli (`brew install heroku/brew/heroku` on Mac) + * Make sure to create a Heroku account if you don't have one, and log in. Local setup: 1. Download the git repository, e.g. using `git clone`. -2. Navigate to be inside the repository base directory. +2. Navigate to be inside the repository base directory. Check also to make sure you're on the correct branch, in case you're using a dev branch instead of master. 3. Start a local pipenv shell with `pipenv shell` -4. Install dependencies with `pipenv install` +4. Run `pipenv install` to install any needed dependencies automatically. 5. Copy the environment variable file using `cp env.example .env` -6. Open the `.env` file and edit to replace `CLIENT_ID` and `CLIENT_SECRET` with your project's matching ID and SECRET (see above) +6. Open the `.env` file (`vi .env`) and edit to replace `CLIENT_ID` and `CLIENT_SECRET` with your project's matching ID and SECRET (see above) 7. Initialize the database locally with `heroku local:run python manage.py migrate` #### Running the app locally Inside the repository base, run: `heroku local`. -The app should be available in a local web browser at: http://127.0.0.1:5000 +The app should be available in a local web browser at: http://127.0.0.1:5000 (if you have amqp; if not, you'll get a 500 Server error. Move on to Heroku deployment below. ) ### Deployment to Heroku -Create a new app in Heroku, and link it to your own repository or use the Heroku-CLI to upload files to the heroku server. +Create a new app in Heroku, and link it to your own repository (connect to Github first inside Heroku) or use the Heroku-CLI to upload files to the heroku server. #### Config in Open Humans @@ -61,9 +62,9 @@ Once you have your heroku app created, go to https://www.openhumans.org/direct-s * `OH_CLIENT_SECRET`: Your Client Secret * `SECRET_KEY` : Something long and random * `PYTHONUNBUFFERED` : true -2. On the Resources tab for your app, edit the Celery Worker to be active. -3. On the same Resources tab, add a CloudAMQP add-on and use the "Little Lemur" version. -4. Deploy your git repository to git (e.g. `git push heroku master`) +2. Deploy your git repository to git (e.g. `git push heroku master`. Note you may need to set the remote, and do a `git pull`, before you can push.) +3. On the Resources tab for your app, edit the Celery Worker to be active. (This may take some time to appear.) +4. On the same Resources tab, add a CloudAMQP add-on and use the "Little Lemur" version. 5. Initialize your database with `heroku run 'python manage.py migrate'` Setup is now complete and the Heroku app should hopefully work!