Feature - Issue 560 - Merges tx-manager-lambda into tx-manager (unfol…

…dingWord-dev#91)

* Merges tx-manager-lambda into tx-manager

* Makes files executable

.coveragerc

@@ -1,13 +1,11 @@
 [run]
 source=
-    client
-    manager
-    converters
-    aws_tools
-    door43_tools
-    general_tools
-    gogs_tools
-    lambda_handlers
+    libraries
+    functions
+
+include =
+    functions/*/main.py
+    libraries/*
 
 omit =
     */__init__.py

.travis.yml

@@ -6,6 +6,8 @@ before install:
   - eval export AWS_ACCESS_KEY_ID=\$${TRAVIS_BRANCH}_AWS_ACCESS_KEY_ID
   - eval export AWS_SECRET_ACCESS_KEY=\$${TRAVIS_BRANCH}_AWS_SECRET_ACCESS_KEY
   - eval export AWS_REGION=\$${TRAVIS_BRANCH}_AWS_REGION
+  - pip install awscli
+  - ./travis-install-apex.sh
 
 install:
   - pip install -r requirements.txt
@@ -15,3 +17,22 @@ script: coverage run test-setup.py test
 
 after_success:
   - coveralls
+
+env:
+  global:
+    - PYTHONDONTWRITEBYTECODE=true
+
+deploy:
+- provider: script
+  skip_cleanup: true
+  script:
+  - ./deploy.sh && ./register-modules.sh https://dev-api.door43.org && ./integration_test.sh
+  on:
+    branch: develop
+- provider: script
+  skip_cleanup: true
+  script:
+  - ./deploy.sh && ./register-modules.sh https://api.door43.org && ./integration_test.sh
+  on:
+    branch: master
+

README.rst

@@ -27,8 +27,6 @@ develop:
 tx-manager
 ==========
 
-This is a python module used with **tx-manager-lambda**. See the `tx-manager-lambda documentation <http://tx-manager-lambda.readthedocs.io/en/latest/readme.html>`_ for details on setting up and deploying the Lambda function.
-
 Project description at `tX Manager Module`_.
 
 Issue for its creation at https://github.com/unfoldingWord-dev/door43.org/issues/53.
@@ -148,3 +146,27 @@ See `tx-md2html_register Lambda function <https://github.com/unfoldingWord-dev/t
 .. _tX Architecture:
 
 .. include:: README-tXArchitecture.rst
+
+
+Setting up as deployed in virtual environment
+=============================================
+
+In IntelliJ terminal, switch to virtual environment and install requirements.
+
+.. highlight:: bash
+    source ~/venv/txml/bin/activate
+    ./install-requirements.sh
+
+Deploying your branch of tx-manager to AWS
+==========================================
+For developing the tx-manager library which this repo uses for every function, you can deploy your code to a test AWS
+environment with apex by doing the following:
+
+* Copy project.test.json.sample to project.test.json
+* Edit project.test.json and change <username> and <branch> to your tx-manager branch
+* Install apex from http://apex.run/#installation
+* Set up your AWS credentials as specified at http://apex.run/#aws-credentials
+* Run `apex deploy --env test` to deploy all functions, or `apex deploy --env test [function-name]` for a single function
+
+For more information on using --env to specify a project json file, see https://github.com/apex/apex/blob/master/docs/projects.md#multiple-environments
+

deploy.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+################################################################################
+#
+# The AWS environment variables will only be available when merging a
+# pull request from develop into master due to travis security settings.
+#
+################################################################################
+
+if [[ ${TRAVIS_EVENT_TYPE} == "push" && (${TRAVIS_BRANCH} == "master" || ${TRAVIS_BRANCH} == "develop") && ${TRAVIS_SECURE_ENV_VARS} == "true" ]]
+then
+    echo "Deploying..."
+    repoDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+    thisDir="$( dirname "${repoDir}" )"
+    "${thisDir}/apex" deploy -C "${repoDir}" --env ${TRAVIS_BRANCH}
+else
+    echo "Not deploying:"
+    echo "  TRAVIS_EVENT_TYPE = $TRAVIS_EVENT_TYPE (must be 'push')"
+    echo "  TRAVIS_BRANCH = $TRAVIS_BRANCH (must be 'master' or 'develop')"
+    echo "  TRAVIS_SECURE_ENV_VARS = $TRAVIS_SECURE_ENV_VARS (must be 'true')"
+fi

functions/client_callback/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/client_callback/.gitignore

@@ -0,0 +1,7 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json

functions/client_callback/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "Callback function for the Web Client, called  by https://api.door43.org/client/callback in the tx_start_job function"
+}

functions/client_callback/main.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.client_callback_handler import ClientCallbackHandler
+
+
+def handle(event, context):
+    """
+    Called by API Gateway when the tx-manager is notified of a finished or failed conversion
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return ClientCallbackHandler().handle(event, context)

functions/client_webhook/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/client_webhook/.gitignore

@@ -0,0 +1,7 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json

functions/client_webhook/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "Client webhook, called by https://api.door43.org/client/webhook from a Gogs repo when commit occurs"
+}

functions/client_webhook/main.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.client_webhook_handler import ClientWebhookHandler
+
+
+def handle(event, context):
+    """
+    Called by API Gateway when a Gogs repo triggers a webhook with API Gateway URL
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return ClientWebhookHandler().handle(event, context)

functions/convert_md2html/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/convert_md2html/.gitignore

@@ -0,0 +1,9 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json
+!/module.json
+

functions/convert_md2html/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "Converts Markdown (MD) files to HTML, called by https://api.door43.org/tx/convert/md2html in the tx_start_job function"
+}

functions/convert_md2html/main.py

@@ -0,0 +1,13 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.convert_handler import ConvertHandler
+from libraries.converters.md2html_converter import Md2HtmlConverter
+
+
+def handle(event, context):
+    """
+    Called through API Gateway to convert a given archive from MD to HTML
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return ConvertHandler(Md2HtmlConverter).handle(event, context)

functions/convert_md2html/module.json

@@ -0,0 +1,11 @@
+{
+    "name": "md2html",
+    "version": "1",
+    "type": "conversion",
+    "resource_types": ["obs", "ta", "tn", "tq", "tw"],
+    "input_format": ["md"],
+    "output_format": ["html"],
+    "options": [],
+    "private_links": [],
+    "public_links": []
+}

functions/convert_usfm2html/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/convert_usfm2html/.gitignore

@@ -0,0 +1,9 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json
+!/module.json
+

functions/convert_usfm2html/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "Converts USFM files to HTML, called by https://api.door43.org/tx/convert/usfm2html in the tx_start_job function"
+}

functions/convert_usfm2html/main.py

@@ -0,0 +1,13 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.convert_handler import ConvertHandler
+from libraries.converters.usfm2html_converter import Usfm2HtmlConverter
+
+
+def handle(event, context):
+    """
+    Called through API Gateway to convert a given archive from MD to HTML
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return ConvertHandler(Usfm2HtmlConverter).handle(event, context)

functions/convert_usfm2html/module.json

@@ -0,0 +1,11 @@
+{
+    "name": "usfm2html",
+    "version": "1",
+    "type": "conversion",
+    "resource_types": ["bible", "ulb", "udb", "reg"],
+    "input_format": ["usfm"],
+    "output_format": ["html"],
+    "options": [],
+    "private_links": [],
+    "public_links": []
+}

functions/dashboard/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/dashboard/.gitignore

@@ -0,0 +1,7 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json

functions/dashboard/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "Dashboard for tX-Manager stats, called from https://api.door43.org/tx/dashboard"
+}

functions/dashboard/main.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.dashboard_handler import DashboardHandler
+
+
+def handle(event, context):
+    """
+    Called by API Gateway when user wants a list of endpoints
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return DashboardHandler().handle(event, context)

functions/door43_deploy/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/door43_deploy/.gitignore

@@ -0,0 +1,7 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json

functions/door43_deploy/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "Deploys a revision of a repo to door43.org, triggered when build_log.json is added to a cdn.door43.org/u/<user>/<repo>/<commit> S3 directory"
+}

functions/door43_deploy/main.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.door43_deploy_handler import Door43DeployHandler
+
+
+def handle(event, context):
+    """
+    Triggered by adding a project's revision directory to the u/ directory of the cdn.door43.org bucket
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return Door43DeployHandler().handle(event, context)

functions/door43_print/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/door43_print/.gitignore

@@ -0,0 +1,7 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json

functions/door43_print/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "Takes  `<user>/<repo>/<commit>` as a parameter and returns a link to a full HTML page for printing. Called by https://api.door43.org/tx/print?id=<user>/<repo>/<commit>"
+}

functions/door43_print/main.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.door43_print_handler import Door43PrintHandler
+
+
+def handle(event, context):
+    """
+    Called by API Gateway when a user wants to print a whole project on live.door43.org
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return Door43PrintHandler().handle(event, context)

functions/list_endpoints/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/list_endpoints/.gitignore

@@ -0,0 +1,7 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json

functions/list_endpoints/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "List endpoints of the tX Manager API, called by https://api.door43.org/tx"
+}

functions/list_endpoints/main.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals, print_function
+from libraries.lambda_handlers.list_endpoints_handler import ListEndpointsHandler
+
+
+def handle(event, context):
+    """
+    Called by API Gateway when user wants a list of endpoints
+    :param dict event:
+    :param context:
+    :return dict:
+    """
+    return ListEndpointsHandler().handle(event, context)

functions/list_jobs/.apexignore

@@ -0,0 +1,4 @@
+*.dist-info/
+*.egg-info/
+.gitignore
+*.pyc

functions/list_jobs/.gitignore

@@ -0,0 +1,7 @@
+*
+!/main.py
+!/.gitignore
+!/.apexignore
+!/__init__.py
+!/requirements.txt
+!/function.json

functions/list_jobs/function.json

@@ -0,0 +1,3 @@
+{
+  "description": "List jobs, called by https://api.door43.org/tx/job[/<job-id>]?gogs_user_token=<user_token> to get the status of one or more jobs"
+}