diff --git a/.coveragerc b/.coveragerc index 31a91e83..e8ee6109 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,13 +1,11 @@ [run] source= - client - manager - converters - aws_tools - door43_tools - general_tools - gogs_tools - lambda_handlers + libraries + functions + +include = + functions/*/main.py + libraries/* omit = */__init__.py diff --git a/.travis.yml b/.travis.yml index 9ae7605a..896114dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ before install: - eval export AWS_ACCESS_KEY_ID=\$${TRAVIS_BRANCH}_AWS_ACCESS_KEY_ID - eval export AWS_SECRET_ACCESS_KEY=\$${TRAVIS_BRANCH}_AWS_SECRET_ACCESS_KEY - eval export AWS_REGION=\$${TRAVIS_BRANCH}_AWS_REGION + - pip install awscli + - ./travis-install-apex.sh install: - pip install -r requirements.txt @@ -15,3 +17,22 @@ script: coverage run test-setup.py test after_success: - coveralls + +env: + global: + - PYTHONDONTWRITEBYTECODE=true + +deploy: +- provider: script + skip_cleanup: true + script: + - ./deploy.sh && ./register-modules.sh https://dev-api.door43.org && ./integration_test.sh + on: + branch: develop +- provider: script + skip_cleanup: true + script: + - ./deploy.sh && ./register-modules.sh https://api.door43.org && ./integration_test.sh + on: + branch: master + diff --git a/README.rst b/README.rst index 5788e215..3eed2152 100644 --- a/README.rst +++ b/README.rst @@ -27,8 +27,6 @@ develop: tx-manager ========== -This is a python module used with **tx-manager-lambda**. See the `tx-manager-lambda documentation `_ for details on setting up and deploying the Lambda function. - Project description at `tX Manager Module`_. Issue for its creation at https://github.com/unfoldingWord-dev/door43.org/issues/53. @@ -148,3 +146,27 @@ See `tx-md2html_register Lambda function and to your tx-manager branch +* Install apex from http://apex.run/#installation +* Set up your AWS credentials as specified at http://apex.run/#aws-credentials +* Run `apex deploy --env test` to deploy all functions, or `apex deploy --env test [function-name]` for a single function + +For more information on using --env to specify a project json file, see https://github.com/apex/apex/blob/master/docs/projects.md#multiple-environments + diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 00000000..ae3fc818 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +################################################################################ +# +# The AWS environment variables will only be available when merging a +# pull request from develop into master due to travis security settings. +# +################################################################################ + +if [[ ${TRAVIS_EVENT_TYPE} == "push" && (${TRAVIS_BRANCH} == "master" || ${TRAVIS_BRANCH} == "develop") && ${TRAVIS_SECURE_ENV_VARS} == "true" ]] +then + echo "Deploying..." + repoDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + thisDir="$( dirname "${repoDir}" )" + "${thisDir}/apex" deploy -C "${repoDir}" --env ${TRAVIS_BRANCH} +else + echo "Not deploying:" + echo " TRAVIS_EVENT_TYPE = $TRAVIS_EVENT_TYPE (must be 'push')" + echo " TRAVIS_BRANCH = $TRAVIS_BRANCH (must be 'master' or 'develop')" + echo " TRAVIS_SECURE_ENV_VARS = $TRAVIS_SECURE_ENV_VARS (must be 'true')" +fi diff --git a/aws_tools/__init__.py b/functions/__init__.py similarity index 100% rename from aws_tools/__init__.py rename to functions/__init__.py diff --git a/functions/client_callback/.apexignore b/functions/client_callback/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/client_callback/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/client_callback/.gitignore b/functions/client_callback/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/client_callback/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/client/__init__.py b/functions/client_callback/__init__.py similarity index 100% rename from client/__init__.py rename to functions/client_callback/__init__.py diff --git a/functions/client_callback/function.json b/functions/client_callback/function.json new file mode 100644 index 00000000..24087288 --- /dev/null +++ b/functions/client_callback/function.json @@ -0,0 +1,3 @@ +{ + "description": "Callback function for the Web Client, called by https://api.door43.org/client/callback in the tx_start_job function" +} diff --git a/functions/client_callback/main.py b/functions/client_callback/main.py new file mode 100644 index 00000000..529268fd --- /dev/null +++ b/functions/client_callback/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.client_callback_handler import ClientCallbackHandler + + +def handle(event, context): + """ + Called by API Gateway when the tx-manager is notified of a finished or failed conversion + :param dict event: + :param context: + :return dict: + """ + return ClientCallbackHandler().handle(event, context) diff --git a/functions/client_webhook/.apexignore b/functions/client_webhook/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/client_webhook/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/client_webhook/.gitignore b/functions/client_webhook/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/client_webhook/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/converters/__init__.py b/functions/client_webhook/__init__.py similarity index 100% rename from converters/__init__.py rename to functions/client_webhook/__init__.py diff --git a/functions/client_webhook/function.json b/functions/client_webhook/function.json new file mode 100644 index 00000000..2740c79f --- /dev/null +++ b/functions/client_webhook/function.json @@ -0,0 +1,3 @@ +{ + "description": "Client webhook, called by https://api.door43.org/client/webhook from a Gogs repo when commit occurs" +} diff --git a/functions/client_webhook/main.py b/functions/client_webhook/main.py new file mode 100644 index 00000000..10c38676 --- /dev/null +++ b/functions/client_webhook/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.client_webhook_handler import ClientWebhookHandler + + +def handle(event, context): + """ + Called by API Gateway when a Gogs repo triggers a webhook with API Gateway URL + :param dict event: + :param context: + :return dict: + """ + return ClientWebhookHandler().handle(event, context) diff --git a/functions/convert_md2html/.apexignore b/functions/convert_md2html/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/convert_md2html/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/convert_md2html/.gitignore b/functions/convert_md2html/.gitignore new file mode 100644 index 00000000..9494fde7 --- /dev/null +++ b/functions/convert_md2html/.gitignore @@ -0,0 +1,9 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json +!/module.json + diff --git a/door43_tools/__init__.py b/functions/convert_md2html/__init__.py similarity index 100% rename from door43_tools/__init__.py rename to functions/convert_md2html/__init__.py diff --git a/functions/convert_md2html/function.json b/functions/convert_md2html/function.json new file mode 100644 index 00000000..cd8d6ba7 --- /dev/null +++ b/functions/convert_md2html/function.json @@ -0,0 +1,3 @@ +{ + "description": "Converts Markdown (MD) files to HTML, called by https://api.door43.org/tx/convert/md2html in the tx_start_job function" +} diff --git a/functions/convert_md2html/main.py b/functions/convert_md2html/main.py new file mode 100644 index 00000000..18972e27 --- /dev/null +++ b/functions/convert_md2html/main.py @@ -0,0 +1,13 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.convert_handler import ConvertHandler +from libraries.converters.md2html_converter import Md2HtmlConverter + + +def handle(event, context): + """ + Called through API Gateway to convert a given archive from MD to HTML + :param dict event: + :param context: + :return dict: + """ + return ConvertHandler(Md2HtmlConverter).handle(event, context) diff --git a/functions/convert_md2html/module.json b/functions/convert_md2html/module.json new file mode 100644 index 00000000..b854adaf --- /dev/null +++ b/functions/convert_md2html/module.json @@ -0,0 +1,11 @@ +{ + "name": "md2html", + "version": "1", + "type": "conversion", + "resource_types": ["obs", "ta", "tn", "tq", "tw"], + "input_format": ["md"], + "output_format": ["html"], + "options": [], + "private_links": [], + "public_links": [] +} diff --git a/functions/convert_usfm2html/.apexignore b/functions/convert_usfm2html/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/convert_usfm2html/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/convert_usfm2html/.gitignore b/functions/convert_usfm2html/.gitignore new file mode 100644 index 00000000..9494fde7 --- /dev/null +++ b/functions/convert_usfm2html/.gitignore @@ -0,0 +1,9 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json +!/module.json + diff --git a/general_tools/__init__.py b/functions/convert_usfm2html/__init__.py similarity index 100% rename from general_tools/__init__.py rename to functions/convert_usfm2html/__init__.py diff --git a/functions/convert_usfm2html/function.json b/functions/convert_usfm2html/function.json new file mode 100644 index 00000000..8c445921 --- /dev/null +++ b/functions/convert_usfm2html/function.json @@ -0,0 +1,3 @@ +{ + "description": "Converts USFM files to HTML, called by https://api.door43.org/tx/convert/usfm2html in the tx_start_job function" +} diff --git a/functions/convert_usfm2html/main.py b/functions/convert_usfm2html/main.py new file mode 100644 index 00000000..2667e7a1 --- /dev/null +++ b/functions/convert_usfm2html/main.py @@ -0,0 +1,13 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.convert_handler import ConvertHandler +from libraries.converters.usfm2html_converter import Usfm2HtmlConverter + + +def handle(event, context): + """ + Called through API Gateway to convert a given archive from MD to HTML + :param dict event: + :param context: + :return dict: + """ + return ConvertHandler(Usfm2HtmlConverter).handle(event, context) diff --git a/functions/convert_usfm2html/module.json b/functions/convert_usfm2html/module.json new file mode 100644 index 00000000..42af1b89 --- /dev/null +++ b/functions/convert_usfm2html/module.json @@ -0,0 +1,11 @@ +{ + "name": "usfm2html", + "version": "1", + "type": "conversion", + "resource_types": ["bible", "ulb", "udb", "reg"], + "input_format": ["usfm"], + "output_format": ["html"], + "options": [], + "private_links": [], + "public_links": [] +} diff --git a/functions/dashboard/.apexignore b/functions/dashboard/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/dashboard/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/dashboard/.gitignore b/functions/dashboard/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/dashboard/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/gogs_tools/__init__.py b/functions/dashboard/__init__.py similarity index 100% rename from gogs_tools/__init__.py rename to functions/dashboard/__init__.py diff --git a/functions/dashboard/function.json b/functions/dashboard/function.json new file mode 100644 index 00000000..034db923 --- /dev/null +++ b/functions/dashboard/function.json @@ -0,0 +1,3 @@ +{ + "description": "Dashboard for tX-Manager stats, called from https://api.door43.org/tx/dashboard" +} diff --git a/functions/dashboard/main.py b/functions/dashboard/main.py new file mode 100644 index 00000000..18733c42 --- /dev/null +++ b/functions/dashboard/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.dashboard_handler import DashboardHandler + + +def handle(event, context): + """ + Called by API Gateway when user wants a list of endpoints + :param dict event: + :param context: + :return dict: + """ + return DashboardHandler().handle(event, context) diff --git a/functions/door43_deploy/.apexignore b/functions/door43_deploy/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/door43_deploy/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/door43_deploy/.gitignore b/functions/door43_deploy/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/door43_deploy/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/lambda_handlers/__init__.py b/functions/door43_deploy/__init__.py similarity index 100% rename from lambda_handlers/__init__.py rename to functions/door43_deploy/__init__.py diff --git a/functions/door43_deploy/function.json b/functions/door43_deploy/function.json new file mode 100644 index 00000000..827f7efb --- /dev/null +++ b/functions/door43_deploy/function.json @@ -0,0 +1,3 @@ +{ + "description": "Deploys a revision of a repo to door43.org, triggered when build_log.json is added to a cdn.door43.org/u/// S3 directory" +} diff --git a/functions/door43_deploy/main.py b/functions/door43_deploy/main.py new file mode 100644 index 00000000..1d0e9068 --- /dev/null +++ b/functions/door43_deploy/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.door43_deploy_handler import Door43DeployHandler + + +def handle(event, context): + """ + Triggered by adding a project's revision directory to the u/ directory of the cdn.door43.org bucket + :param dict event: + :param context: + :return dict: + """ + return Door43DeployHandler().handle(event, context) diff --git a/functions/door43_print/.apexignore b/functions/door43_print/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/door43_print/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/door43_print/.gitignore b/functions/door43_print/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/door43_print/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/manager/__init__.py b/functions/door43_print/__init__.py similarity index 100% rename from manager/__init__.py rename to functions/door43_print/__init__.py diff --git a/functions/door43_print/function.json b/functions/door43_print/function.json new file mode 100644 index 00000000..9478967c --- /dev/null +++ b/functions/door43_print/function.json @@ -0,0 +1,3 @@ +{ + "description": "Takes `//` as a parameter and returns a link to a full HTML page for printing. Called by https://api.door43.org/tx/print?id=//" +} diff --git a/functions/door43_print/main.py b/functions/door43_print/main.py new file mode 100644 index 00000000..992a41e2 --- /dev/null +++ b/functions/door43_print/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.door43_print_handler import Door43PrintHandler + + +def handle(event, context): + """ + Called by API Gateway when a user wants to print a whole project on live.door43.org + :param dict event: + :param context: + :return dict: + """ + return Door43PrintHandler().handle(event, context) diff --git a/functions/list_endpoints/.apexignore b/functions/list_endpoints/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/list_endpoints/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/list_endpoints/.gitignore b/functions/list_endpoints/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/list_endpoints/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/resource_container/__init__.py b/functions/list_endpoints/__init__.py similarity index 100% rename from resource_container/__init__.py rename to functions/list_endpoints/__init__.py diff --git a/functions/list_endpoints/function.json b/functions/list_endpoints/function.json new file mode 100644 index 00000000..c616f3cc --- /dev/null +++ b/functions/list_endpoints/function.json @@ -0,0 +1,3 @@ +{ + "description": "List endpoints of the tX Manager API, called by https://api.door43.org/tx" +} diff --git a/functions/list_endpoints/main.py b/functions/list_endpoints/main.py new file mode 100644 index 00000000..96688130 --- /dev/null +++ b/functions/list_endpoints/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.list_endpoints_handler import ListEndpointsHandler + + +def handle(event, context): + """ + Called by API Gateway when user wants a list of endpoints + :param dict event: + :param context: + :return dict: + """ + return ListEndpointsHandler().handle(event, context) diff --git a/functions/list_jobs/.apexignore b/functions/list_jobs/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/list_jobs/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/list_jobs/.gitignore b/functions/list_jobs/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/list_jobs/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/functions/list_jobs/__init__.py b/functions/list_jobs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/functions/list_jobs/function.json b/functions/list_jobs/function.json new file mode 100644 index 00000000..f8d0dad7 --- /dev/null +++ b/functions/list_jobs/function.json @@ -0,0 +1,3 @@ +{ + "description": "List jobs, called by https://api.door43.org/tx/job[/]?gogs_user_token= to get the status of one or more jobs" +} diff --git a/functions/list_jobs/main.py b/functions/list_jobs/main.py new file mode 100644 index 00000000..0578528c --- /dev/null +++ b/functions/list_jobs/main.py @@ -0,0 +1,14 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.list_jobs_handler import ListJobsHandler + + +def handle(event, context): + """ + Called by the wbhook client to list jobs to get their status + :param dict event: + :param context: + :return dict: + """ + print("EVENT:") + print(event) + return ListJobsHandler().handle(event, context) diff --git a/functions/register_module/.apexignore b/functions/register_module/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/register_module/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/register_module/.gitignore b/functions/register_module/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/register_module/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/functions/register_module/__init__.py b/functions/register_module/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/functions/register_module/function.json b/functions/register_module/function.json new file mode 100644 index 00000000..9d284b6e --- /dev/null +++ b/functions/register_module/function.json @@ -0,0 +1,3 @@ +{ + "description": "Registers a conversion module, called by https://api.door43.org/tx/module" +} diff --git a/functions/register_module/main.py b/functions/register_module/main.py new file mode 100644 index 00000000..790a06e6 --- /dev/null +++ b/functions/register_module/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.register_module_handler import RegisterModuleHandler + + +def handle(event, context): + """ + Called by a module when it is deployed to register it + :param dict event: + :param context: + :return dict: + """ + return RegisterModuleHandler().handle(event, context) diff --git a/functions/request_job/.apexignore b/functions/request_job/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/request_job/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/request_job/.gitignore b/functions/request_job/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/request_job/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/functions/request_job/__init__.py b/functions/request_job/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/functions/request_job/function.json b/functions/request_job/function.json new file mode 100644 index 00000000..5de8520f --- /dev/null +++ b/functions/request_job/function.json @@ -0,0 +1,3 @@ +{ + "description": "Requests a job, called by https://api.door43.org/tx/job in the tx_client_webhook function" +} diff --git a/functions/request_job/main.py b/functions/request_job/main.py new file mode 100644 index 00000000..6d0ef20c --- /dev/null +++ b/functions/request_job/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.request_job_handler import RequestJobHandler + + +def handle(event, context): + """ + Called by the webhook client to request a job + :param dict event: + :param context: + :return dict: + """ + return RequestJobHandler().handle(event, context) diff --git a/functions/start_job/.apexignore b/functions/start_job/.apexignore new file mode 100644 index 00000000..b7e45044 --- /dev/null +++ b/functions/start_job/.apexignore @@ -0,0 +1,4 @@ +*.dist-info/ +*.egg-info/ +.gitignore +*.pyc diff --git a/functions/start_job/.gitignore b/functions/start_job/.gitignore new file mode 100644 index 00000000..93be21eb --- /dev/null +++ b/functions/start_job/.gitignore @@ -0,0 +1,7 @@ +* +!/main.py +!/.gitignore +!/.apexignore +!/__init__.py +!/requirements.txt +!/function.json diff --git a/functions/start_job/__init__.py b/functions/start_job/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/functions/start_job/function.json b/functions/start_job/function.json new file mode 100644 index 00000000..a9801990 --- /dev/null +++ b/functions/start_job/function.json @@ -0,0 +1,3 @@ +{ + "description": "Starts a job, triggered when a row to the DynamoDB tx-job table is inserted" +} diff --git a/functions/start_job/main.py b/functions/start_job/main.py new file mode 100644 index 00000000..7b77f47d --- /dev/null +++ b/functions/start_job/main.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals, print_function +from libraries.lambda_handlers.start_job_handler import StartJobHandler + + +def handle(event, context): + """ + Triggered by adding a file to the cdn.door43.org/temp S3 folder + :param dict event: + :param context: + :return dict: + """ + return StartJobHandler().handle(event, context) diff --git a/integration_test.sh b/integration_test.sh new file mode 100755 index 00000000..28d83a7f --- /dev/null +++ b/integration_test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +export TEST_DEPLOYED=test_deployed +python -m unittest -v tests.integration_tests.test_conversion + +echo "Completed Successfully" diff --git a/libraries/__init__.py b/libraries/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libraries/aws_tools/__init__.py b/libraries/aws_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/aws_tools/dynamodb_handler.py b/libraries/aws_tools/dynamodb_handler.py similarity index 99% rename from aws_tools/dynamodb_handler.py rename to libraries/aws_tools/dynamodb_handler.py index b9f9218a..9509748e 100644 --- a/aws_tools/dynamodb_handler.py +++ b/libraries/aws_tools/dynamodb_handler.py @@ -3,7 +3,7 @@ import logging from six import iteritems from boto3 import Session -from boto3.dynamodb.conditions import Attr, Key +from boto3.dynamodb.conditions import Attr class DynamoDBHandler(object): diff --git a/aws_tools/s3_handler.py b/libraries/aws_tools/s3_handler.py similarity index 99% rename from aws_tools/s3_handler.py rename to libraries/aws_tools/s3_handler.py index b2028e9b..40a6d783 100644 --- a/aws_tools/s3_handler.py +++ b/libraries/aws_tools/s3_handler.py @@ -4,7 +4,7 @@ import boto3 import botocore from boto3.session import Session -from general_tools.file_utils import get_mime_type +from libraries.general_tools.file_utils import get_mime_type class S3Handler(object): diff --git a/libraries/client/__init__.py b/libraries/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/client/client_callback.py b/libraries/client/client_callback.py similarity index 95% rename from client/client_callback.py rename to libraries/client/client_callback.py index 30f7f49e..b0b25094 100644 --- a/client/client_callback.py +++ b/libraries/client/client_callback.py @@ -4,10 +4,10 @@ import logging import time from logging import Logger -from general_tools.file_utils import unzip, write_file -from general_tools.url_utils import download_file -from aws_tools.s3_handler import S3Handler -from manager.job import TxJob +from libraries.general_tools.file_utils import unzip, write_file +from libraries.general_tools.url_utils import download_file +from libraries.aws_tools.s3_handler import S3Handler +from libraries.manager.job import TxJob class ClientCallback(object): diff --git a/client/client_webhook.py b/libraries/client/client_webhook.py similarity index 96% rename from client/client_webhook.py rename to libraries/client/client_webhook.py index a501ccb6..f00d6b81 100644 --- a/client/client_webhook.py +++ b/libraries/client/client_webhook.py @@ -5,11 +5,11 @@ import logging import json from datetime import datetime -from general_tools.file_utils import unzip, get_subdirs, write_file, add_contents_to_zip, add_file_to_zip -from general_tools.url_utils import download_file -from resource_container.ResourceContainer import RC -from client.preprocessors import do_preprocess -from aws_tools.s3_handler import S3Handler +from libraries.general_tools.file_utils import unzip, get_subdirs, write_file, add_contents_to_zip, add_file_to_zip +from libraries.general_tools.url_utils import download_file +from libraries.resource_container.ResourceContainer import RC +from libraries.client.preprocessors import do_preprocess +from libraries.aws_tools.s3_handler import S3Handler class ClientWebhook(object): diff --git a/client/preprocessors.py b/libraries/client/preprocessors.py similarity index 98% rename from client/preprocessors.py rename to libraries/client/preprocessors.py index aa9194d5..9945ebd0 100644 --- a/client/preprocessors.py +++ b/libraries/client/preprocessors.py @@ -1,12 +1,12 @@ from __future__ import unicode_literals, print_function import os import re -from door43_tools import bible_books -from general_tools.file_utils import write_file, read_file +from libraries.door43_tools import bible_books +from libraries.general_tools.file_utils import write_file, read_file from shutil import copy -from resource_container.ResourceContainer import RC +from libraries.resource_container.ResourceContainer import RC from glob import glob -from resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES +from libraries.resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES def do_preprocess(rc, repo_dir, output_dir): diff --git a/libraries/converters/__init__.py b/libraries/converters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/convert_logger.py b/libraries/converters/convert_logger.py similarity index 100% rename from converters/convert_logger.py rename to libraries/converters/convert_logger.py diff --git a/converters/converter.py b/libraries/converters/converter.py similarity index 95% rename from converters/converter.py rename to libraries/converters/converter.py index 51612142..35c4c626 100644 --- a/converters/converter.py +++ b/libraries/converters/converter.py @@ -1,9 +1,9 @@ from __future__ import print_function, unicode_literals import os import tempfile -from aws_tools.s3_handler import S3Handler -from general_tools.url_utils import download_file -from general_tools.file_utils import unzip, add_contents_to_zip, remove_tree, remove +from libraries.aws_tools.s3_handler import S3Handler +from libraries.general_tools.url_utils import download_file +from libraries.general_tools.file_utils import unzip, add_contents_to_zip, remove_tree, remove from shutil import copy import logging from convert_logger import ConvertLogger diff --git a/converters/md2html_converter.py b/libraries/converters/md2html_converter.py similarity index 96% rename from converters/md2html_converter.py rename to libraries/converters/md2html_converter.py index f9740d25..8d035861 100644 --- a/converters/md2html_converter.py +++ b/libraries/converters/md2html_converter.py @@ -5,10 +5,10 @@ import codecs from shutil import copyfile from bs4 import BeautifulSoup -from general_tools.file_utils import write_file +from libraries.general_tools.file_utils import write_file from converter import Converter -from door43_tools.obs_handler import OBSInspection -from door43_tools.obs_data import obs_data +from libraries.door43_tools.obs_handler import OBSInspection +from libraries.door43_tools.obs_data import obs_data class Md2HtmlConverter(Converter): diff --git a/converters/templates/template.html b/libraries/converters/templates/template.html similarity index 100% rename from converters/templates/template.html rename to libraries/converters/templates/template.html diff --git a/converters/usfm2html_converter.py b/libraries/converters/usfm2html_converter.py similarity index 95% rename from converters/usfm2html_converter.py rename to libraries/converters/usfm2html_converter.py index f49a3878..252c7aa0 100644 --- a/converters/usfm2html_converter.py +++ b/libraries/converters/usfm2html_converter.py @@ -4,10 +4,10 @@ import codecs from bs4 import BeautifulSoup from shutil import copyfile -from general_tools.file_utils import write_file, remove_tree +from libraries.general_tools.file_utils import write_file, remove_tree from converter import Converter from usfm_tools.transform import UsfmTransform -from resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES +from libraries.resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES class Usfm2HtmlConverter(Converter): diff --git a/libraries/door43_tools/__init__.py b/libraries/door43_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/door43_tools/bible_books.py b/libraries/door43_tools/bible_books.py similarity index 100% rename from door43_tools/bible_books.py rename to libraries/door43_tools/bible_books.py diff --git a/door43_tools/obs_data.py b/libraries/door43_tools/obs_data.py similarity index 100% rename from door43_tools/obs_data.py rename to libraries/door43_tools/obs_data.py diff --git a/door43_tools/obs_handler.py b/libraries/door43_tools/obs_handler.py similarity index 97% rename from door43_tools/obs_handler.py rename to libraries/door43_tools/obs_handler.py index 44e044e1..1fd243fc 100644 --- a/door43_tools/obs_handler.py +++ b/libraries/door43_tools/obs_handler.py @@ -1,7 +1,7 @@ from __future__ import print_function, unicode_literals import os from bs4 import BeautifulSoup -from converters.convert_logger import ConvertLogger +from libraries.converters.convert_logger import ConvertLogger from obs_data import obs_data diff --git a/door43_tools/project_deployer.py b/libraries/door43_tools/project_deployer.py similarity index 97% rename from door43_tools/project_deployer.py rename to libraries/door43_tools/project_deployer.py index d0a8a156..58922ede 100644 --- a/door43_tools/project_deployer.py +++ b/libraries/door43_tools/project_deployer.py @@ -6,9 +6,9 @@ import logging from glob import glob from shutil import copyfile -from aws_tools.s3_handler import S3Handler -from general_tools.file_utils import write_file -from door43_tools.templaters import do_template +from libraries.aws_tools.s3_handler import S3Handler +from libraries.general_tools.file_utils import write_file +from libraries.door43_tools.templaters import do_template from datetime import datetime, timedelta diff --git a/door43_tools/project_printer.py b/libraries/door43_tools/project_printer.py similarity index 94% rename from door43_tools/project_printer.py rename to libraries/door43_tools/project_printer.py index e6fdfe05..93160c66 100644 --- a/door43_tools/project_printer.py +++ b/libraries/door43_tools/project_printer.py @@ -4,9 +4,9 @@ import codecs from bs4 import BeautifulSoup from glob import glob -from aws_tools.s3_handler import S3Handler -from general_tools.file_utils import read_file -from resource_container.ResourceContainer import RC +from libraries.aws_tools.s3_handler import S3Handler +from libraries.general_tools.file_utils import read_file +from libraries.resource_container.ResourceContainer import RC class ProjectPrinter(object): diff --git a/door43_tools/td_language.py b/libraries/door43_tools/td_language.py similarity index 96% rename from door43_tools/td_language.py rename to libraries/door43_tools/td_language.py index cae5fa4a..8133392d 100644 --- a/door43_tools/td_language.py +++ b/libraries/door43_tools/td_language.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals import json -from general_tools import url_utils +from libraries.general_tools import url_utils class TdLanguage(object): diff --git a/door43_tools/templaters.py b/libraries/door43_tools/templaters.py similarity index 97% rename from door43_tools/templaters.py rename to libraries/door43_tools/templaters.py index ab13a540..1044556b 100644 --- a/door43_tools/templaters.py +++ b/libraries/door43_tools/templaters.py @@ -4,10 +4,10 @@ import logging from glob import glob from bs4 import BeautifulSoup -from general_tools.file_utils import write_file -from resource_container.ResourceContainer import RC -from general_tools.file_utils import load_yaml_object -from resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES +from libraries.general_tools.file_utils import write_file +from libraries.resource_container.ResourceContainer import RC +from libraries.general_tools.file_utils import load_yaml_object +from libraries.resource_container.ResourceContainer import BIBLE_RESOURCE_TYPES def do_template(resource_type, source_dir, output_dir, template_file): diff --git a/libraries/general_tools/__init__.py b/libraries/general_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/general_tools/file_utils.py b/libraries/general_tools/file_utils.py similarity index 100% rename from general_tools/file_utils.py rename to libraries/general_tools/file_utils.py diff --git a/general_tools/print_utils.py b/libraries/general_tools/print_utils.py similarity index 100% rename from general_tools/print_utils.py rename to libraries/general_tools/print_utils.py diff --git a/general_tools/smartquotes.py b/libraries/general_tools/smartquotes.py similarity index 100% rename from general_tools/smartquotes.py rename to libraries/general_tools/smartquotes.py diff --git a/general_tools/url_utils.py b/libraries/general_tools/url_utils.py similarity index 100% rename from general_tools/url_utils.py rename to libraries/general_tools/url_utils.py diff --git a/libraries/gogs_tools/__init__.py b/libraries/gogs_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gogs_tools/gogs_handler.py b/libraries/gogs_tools/gogs_handler.py similarity index 100% rename from gogs_tools/gogs_handler.py rename to libraries/gogs_tools/gogs_handler.py diff --git a/libraries/lambda_handlers/__init__.py b/libraries/lambda_handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lambda_handlers/client_callback_handler.py b/libraries/lambda_handlers/client_callback_handler.py similarity index 87% rename from lambda_handlers/client_callback_handler.py rename to libraries/lambda_handlers/client_callback_handler.py index 4eff22ed..d770f47b 100644 --- a/lambda_handlers/client_callback_handler.py +++ b/libraries/lambda_handlers/client_callback_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from lambda_handlers.handler import Handler -from client.client_callback import ClientCallback +from libraries.lambda_handlers.handler import Handler +from libraries.client.client_callback import ClientCallback class ClientCallbackHandler(Handler): diff --git a/lambda_handlers/client_webhook_handler.py b/libraries/lambda_handlers/client_webhook_handler.py similarity index 90% rename from lambda_handlers/client_webhook_handler.py rename to libraries/lambda_handlers/client_webhook_handler.py index 4e7e831e..7431946d 100644 --- a/lambda_handlers/client_webhook_handler.py +++ b/libraries/lambda_handlers/client_webhook_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from lambda_handlers.handler import Handler -from client.client_webhook import ClientWebhook +from libraries.lambda_handlers.handler import Handler +from libraries.client.client_webhook import ClientWebhook class ClientWebhookHandler(Handler): diff --git a/lambda_handlers/convert_handler.py b/libraries/lambda_handlers/convert_handler.py similarity index 96% rename from lambda_handlers/convert_handler.py rename to libraries/lambda_handlers/convert_handler.py index f26dee13..5ee0b1a9 100644 --- a/lambda_handlers/convert_handler.py +++ b/libraries/lambda_handlers/convert_handler.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals, print_function -from lambda_handlers.handler import Handler +from libraries.lambda_handlers.handler import Handler class ConvertHandler(Handler): diff --git a/lambda_handlers/dashboard_handler.py b/libraries/lambda_handlers/dashboard_handler.py similarity index 92% rename from lambda_handlers/dashboard_handler.py rename to libraries/lambda_handlers/dashboard_handler.py index 86751e3a..d4b0f93d 100644 --- a/lambda_handlers/dashboard_handler.py +++ b/libraries/lambda_handlers/dashboard_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from manager.manager import TxManager -from lambda_handlers.handler import Handler +from libraries.manager.manager import TxManager +from libraries.lambda_handlers.handler import Handler class DashboardHandler(Handler): diff --git a/lambda_handlers/door43_deploy_handler.py b/libraries/lambda_handlers/door43_deploy_handler.py similarity index 93% rename from lambda_handlers/door43_deploy_handler.py rename to libraries/lambda_handlers/door43_deploy_handler.py index 73e08c57..da4375bc 100644 --- a/lambda_handlers/door43_deploy_handler.py +++ b/libraries/lambda_handlers/door43_deploy_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from door43_tools.project_deployer import ProjectDeployer -from lambda_handlers.handler import Handler +from libraries.door43_tools.project_deployer import ProjectDeployer +from libraries.lambda_handlers.handler import Handler class Door43DeployHandler(Handler): diff --git a/lambda_handlers/door43_print_handler.py b/libraries/lambda_handlers/door43_print_handler.py similarity index 87% rename from lambda_handlers/door43_print_handler.py rename to libraries/lambda_handlers/door43_print_handler.py index 3d2bb1b1..6f60046a 100644 --- a/lambda_handlers/door43_print_handler.py +++ b/libraries/lambda_handlers/door43_print_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from door43_tools.project_printer import ProjectPrinter -from lambda_handlers.handler import Handler +from libraries.door43_tools.project_printer import ProjectPrinter +from libraries.lambda_handlers.handler import Handler class Door43PrintHandler(Handler): diff --git a/lambda_handlers/handler.py b/libraries/lambda_handlers/handler.py similarity index 100% rename from lambda_handlers/handler.py rename to libraries/lambda_handlers/handler.py diff --git a/lambda_handlers/list_endpoints_handler.py b/libraries/lambda_handlers/list_endpoints_handler.py similarity index 91% rename from lambda_handlers/list_endpoints_handler.py rename to libraries/lambda_handlers/list_endpoints_handler.py index b145283c..6810f9bd 100644 --- a/lambda_handlers/list_endpoints_handler.py +++ b/libraries/lambda_handlers/list_endpoints_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from manager.manager import TxManager -from lambda_handlers.handler import Handler +from libraries.manager.manager import TxManager +from libraries.lambda_handlers.handler import Handler class ListEndpointsHandler(Handler): diff --git a/lambda_handlers/list_jobs_handler.py b/libraries/lambda_handlers/list_jobs_handler.py similarity index 91% rename from lambda_handlers/list_jobs_handler.py rename to libraries/lambda_handlers/list_jobs_handler.py index 7cd83022..ba9b2251 100644 --- a/lambda_handlers/list_jobs_handler.py +++ b/libraries/lambda_handlers/list_jobs_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from manager.manager import TxManager -from lambda_handlers.handler import Handler +from libraries.manager.manager import TxManager +from libraries.lambda_handlers.handler import Handler class ListJobsHandler(Handler): diff --git a/lambda_handlers/register_module_handler.py b/libraries/lambda_handlers/register_module_handler.py similarity index 91% rename from lambda_handlers/register_module_handler.py rename to libraries/lambda_handlers/register_module_handler.py index a36a3dfb..c541be6f 100644 --- a/lambda_handlers/register_module_handler.py +++ b/libraries/lambda_handlers/register_module_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from manager.manager import TxManager -from lambda_handlers.handler import Handler +from libraries.manager.manager import TxManager +from libraries.lambda_handlers.handler import Handler class RegisterModuleHandler(Handler): diff --git a/lambda_handlers/request_job_handler.py b/libraries/lambda_handlers/request_job_handler.py similarity index 92% rename from lambda_handlers/request_job_handler.py rename to libraries/lambda_handlers/request_job_handler.py index 3176cc0f..55e20ebd 100644 --- a/lambda_handlers/request_job_handler.py +++ b/libraries/lambda_handlers/request_job_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from manager.manager import TxManager -from lambda_handlers.handler import Handler +from libraries.manager.manager import TxManager +from libraries.lambda_handlers.handler import Handler class RequestJobHandler(Handler): diff --git a/lambda_handlers/start_job_handler.py b/libraries/lambda_handlers/start_job_handler.py similarity index 90% rename from lambda_handlers/start_job_handler.py rename to libraries/lambda_handlers/start_job_handler.py index f1bdfdea..40e70c27 100644 --- a/lambda_handlers/start_job_handler.py +++ b/libraries/lambda_handlers/start_job_handler.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals, print_function -from manager.manager import TxManager -from lambda_handlers.handler import Handler +from libraries.manager.manager import TxManager +from libraries.lambda_handlers.handler import Handler class StartJobHandler(Handler): diff --git a/libraries/manager/__init__.py b/libraries/manager/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/manager/job.py b/libraries/manager/job.py similarity index 100% rename from manager/job.py rename to libraries/manager/job.py diff --git a/manager/manager.py b/libraries/manager/manager.py similarity index 99% rename from manager/manager.py rename to libraries/manager/manager.py index d09a85e6..fa007936 100644 --- a/manager/manager.py +++ b/libraries/manager/manager.py @@ -6,8 +6,8 @@ from datetime import datetime from datetime import timedelta from bs4 import BeautifulSoup -from aws_tools.dynamodb_handler import DynamoDBHandler -from gogs_tools.gogs_handler import GogsHandler +from libraries.aws_tools.dynamodb_handler import DynamoDBHandler +from libraries.gogs_tools.gogs_handler import GogsHandler from job import TxJob from module import TxModule diff --git a/manager/module.py b/libraries/manager/module.py similarity index 100% rename from manager/module.py rename to libraries/manager/module.py diff --git a/manager/object.py b/libraries/manager/object.py similarity index 100% rename from manager/object.py rename to libraries/manager/object.py diff --git a/resource_container/ResourceContainer.py b/libraries/resource_container/ResourceContainer.py similarity index 99% rename from resource_container/ResourceContainer.py rename to libraries/resource_container/ResourceContainer.py index 0e121f68..2440936f 100644 --- a/resource_container/ResourceContainer.py +++ b/libraries/resource_container/ResourceContainer.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals import os import re -from door43_tools.td_language import TdLanguage -from door43_tools.bible_books import BOOK_NAMES -from general_tools.file_utils import load_json_object, load_yaml_object, read_file +from libraries.door43_tools.td_language import TdLanguage +from libraries.door43_tools.bible_books import BOOK_NAMES +from libraries.general_tools.file_utils import load_json_object, load_yaml_object, read_file from datetime import datetime from glob import glob diff --git a/libraries/resource_container/__init__.py b/libraries/resource_container/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project.develop.json b/project.develop.json new file mode 100644 index 00000000..4ce6b397 --- /dev/null +++ b/project.develop.json @@ -0,0 +1,12 @@ +{ + "name": "dev-tx", + "description": "DEVELOPMENT tX Manager lambda functions for handling conversions", + "memory": 512, + "timeout": 300, + "role": "arn:aws:iam::002723143144:role/dev-tx_lambda_function", + "environment": {}, + "hooks": { + "build": "pip install --upgrade -r ../../requirements.txt -t .;cp -Rf ../../libraries .", + "clean": "rm -rf libraries" + } +} diff --git a/project.master.json b/project.master.json new file mode 100644 index 00000000..87f48346 --- /dev/null +++ b/project.master.json @@ -0,0 +1,12 @@ +{ + "name": "tx", + "description": "tX Manager lambda functions for handling conversions", + "memory": 512, + "timeout": 300, + "role": "arn:aws:iam::002723143144:role/tx_lambda_function", + "environment": {}, + "hooks": { + "build": "pip install --upgrade -r ../../requirements.txt -t .;cp -Rf ../../libraries .", + "clean": "rm -rf libraries" + } +} diff --git a/project.test.json b/project.test.json new file mode 100644 index 00000000..aa347dfd --- /dev/null +++ b/project.test.json @@ -0,0 +1,12 @@ +{ + "name": "test-tx", + "description": "Testing project for tX Manager", + "memory": 512, + "timeout": 300, + "role": "arn:aws:iam::581647696645:role/tx_lambda_function", + "environment": {}, + "hooks": { + "build": "pip install --upgrade -r ../../requirements.txt -t .;cp -Rf ../../libraries .", + "clean": "rm -rf libraries" + } +} diff --git a/register-modules.sh b/register-modules.sh new file mode 100755 index 00000000..a4fa9145 --- /dev/null +++ b/register-modules.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +API_URL=$1 + +#register md2html +curl --header "Content-Type: application/json" -X POST --data @functions/convert_md2html/module.json "${API_URL}/tx/module" + +#register usfm2html +curl --header "Content-Type: application/json" -X POST --data @functions/convert_usfm2html/module.json "${API_URL}/tx/module" + diff --git a/setup.py b/setup.py index bb1cd864..b6082ff5 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,37 @@ def read(f_name): setup( name='tx-manager', - version='0.2.64', + version='0.2.65', + package_dir={ + 'client_callback': 'functions/client_callback', + 'client_webhook': 'functions/client_webhook', + 'convert_md2html': 'functions/convert_md2html', + 'convert_usfm2html': 'functions/convert_usfm2html', + 'door43_deploy': 'functions/door43_deploy', + 'list_endpoints': 'functions/list_endpoints', + 'register_module': 'functions/register_module', + 'request_job': 'functions/request_job', + 'start_job': 'functions/start_job', + 'client': 'libraries/client', + 'converters': 'libraries/converters', + 'aws_tools': 'libraries/aws_tools', + 'door43_tools': 'libraries/door43_tools', + 'general_tools': 'libraries/general_tools', + 'gogs_tools': 'libraries/gogs_tools', + 'lambda_handlers': 'libraries/lambda_handlers', + 'manager': 'libraries/manager', + 'resource_container': 'libraries/resource_container' + }, packages=[ + 'client_callback', + 'client_webhook', + 'convert_md2html', + 'convert_usfm2html', + 'door43_deploy', + 'list_endpoints', + 'register_module', + 'request_job', + 'start_job', 'client', 'converters', 'aws_tools', diff --git a/test-setup.py b/test-setup.py index 577cfec5..16352352 100644 --- a/test-setup.py +++ b/test-setup.py @@ -2,8 +2,37 @@ setup( name='tx-manager', - version='0.2.64', + version='0.2.65', + package_dir={ + 'client_callback': 'functions/client_callback', + 'client_webhook': 'functions/client_webhook', + 'convert_md2html': 'functions/convert_md2html', + 'convert_usfm2html': 'functions/convert_usfm2html', + 'door43_deploy': 'functions/door43_deploy', + 'list_endpoints': 'functions/list_endpoints', + 'register_module': 'functions/register_module', + 'request_job': 'functions/request_job', + 'start_job': 'functions/start_job', + 'client': 'libraries/client', + 'converters': 'libraries/converters', + 'aws_tools': 'libraries/aws_tools', + 'door43_tools': 'libraries/door43_tools', + 'general_tools': 'libraries/general_tools', + 'gogs_tools': 'libraries/gogs_tools', + 'lambda_handlers': 'libraries/lambda_handlers', + 'manager': 'libraries/manager', + 'resource_container': 'libraries/resource_container' + }, packages=[ + 'client_callback', + 'client_webhook', + 'convert_md2html', + 'convert_usfm2html', + 'door43_deploy', + 'list_endpoints', + 'register_module', + 'request_job', + 'start_job', 'client', 'converters', 'aws_tools', diff --git a/tests/aws_tools_tests/test_dynamodbHandler.py b/tests/aws_tools_tests/test_dynamodbHandler.py index 947cd609..b0ae43b4 100644 --- a/tests/aws_tools_tests/test_dynamodbHandler.py +++ b/tests/aws_tools_tests/test_dynamodbHandler.py @@ -1,15 +1,15 @@ from __future__ import absolute_import, unicode_literals, print_function import mock import unittest -import aws_tools.dynamodb_handler +from libraries.aws_tools.dynamodb_handler import DynamoDBHandler class DynamoDBHandlerTests(unittest.TestCase): @classmethod def setUpClass(cls): - with mock.patch("aws_tools.dynamodb_handler.boto3", mock.MagicMock()): - cls.handler = aws_tools.dynamodb_handler.DynamoDBHandler("table_name") + with mock.patch("libraries.aws_tools.dynamodb_handler.boto3", mock.MagicMock()): + cls.handler = DynamoDBHandler("table_name") cls.handler.table = mock.MagicMock() def setUp(self): diff --git a/tests/aws_tools_tests/test_s3_handler.py b/tests/aws_tools_tests/test_s3_handler.py index 46c5771c..66814467 100644 --- a/tests/aws_tools_tests/test_s3_handler.py +++ b/tests/aws_tools_tests/test_s3_handler.py @@ -4,7 +4,7 @@ import tempfile from botocore.exceptions import ClientError from unittest import TestCase -from aws_tools.s3_handler import S3Handler +from libraries.aws_tools.s3_handler import S3Handler from moto import mock_s3 diff --git a/tests/client_tests/test_bible_preprocessor.py b/tests/client_tests/test_bible_preprocessor.py index d0c1f001..8c0457f4 100644 --- a/tests/client_tests/test_bible_preprocessor.py +++ b/tests/client_tests/test_bible_preprocessor.py @@ -4,9 +4,9 @@ import tempfile import unittest import shutil -from resource_container.ResourceContainer import RC -from client.preprocessors import do_preprocess -from general_tools.file_utils import unzip +from libraries.resource_container.ResourceContainer import RC +from libraries.client.preprocessors import do_preprocess +from libraries.general_tools.file_utils import unzip class TestBiblePreprocessor(unittest.TestCase): diff --git a/tests/client_tests/test_client_webhook.py b/tests/client_tests/test_client_webhook.py index 0cfc1ba5..2597bcaa 100644 --- a/tests/client_tests/test_client_webhook.py +++ b/tests/client_tests/test_client_webhook.py @@ -4,7 +4,7 @@ import tempfile import unittest from mock import patch -from client.client_webhook import ClientWebhook +from libraries.client.client_webhook import ClientWebhook class TestClientWebhook(unittest.TestCase): @@ -29,7 +29,7 @@ def tearDown(self): if os.path.isdir(self.temp_dir): shutil.rmtree(self.temp_dir, ignore_errors=True) - @patch('client.client_webhook.download_file') + @patch('libraries.client.client_webhook.download_file') def test_download_repo(self, mock_download_file): mock_download_file.side_effect = self.mock_download_repo cwh = ClientWebhook() diff --git a/tests/client_tests/test_obs_preprocessor.py b/tests/client_tests/test_obs_preprocessor.py index d6b7600c..c5c4330c 100644 --- a/tests/client_tests/test_obs_preprocessor.py +++ b/tests/client_tests/test_obs_preprocessor.py @@ -3,9 +3,9 @@ import tempfile import unittest import shutil -from resource_container.ResourceContainer import RC -from client.preprocessors import do_preprocess -from general_tools.file_utils import unzip, add_contents_to_zip +from libraries.resource_container.ResourceContainer import RC +from libraries.client.preprocessors import do_preprocess +from libraries.general_tools.file_utils import unzip, add_contents_to_zip class TestObsPreprocessor(unittest.TestCase): diff --git a/tests/client_tests/test_preprocessor.py b/tests/client_tests/test_preprocessor.py index acc50c25..c7635f17 100644 --- a/tests/client_tests/test_preprocessor.py +++ b/tests/client_tests/test_preprocessor.py @@ -4,9 +4,9 @@ import unittest import shutil import markdown -from resource_container.ResourceContainer import RC -from client.preprocessors import do_preprocess -from general_tools.file_utils import unzip, read_file +from libraries.resource_container.ResourceContainer import RC +from libraries.client.preprocessors import do_preprocess +from libraries.general_tools.file_utils import unzip, read_file from bs4 import BeautifulSoup diff --git a/tests/client_tests/test_ta_preprocessor.py b/tests/client_tests/test_ta_preprocessor.py index 94cd2198..8cff0b0d 100644 --- a/tests/client_tests/test_ta_preprocessor.py +++ b/tests/client_tests/test_ta_preprocessor.py @@ -4,9 +4,9 @@ import unittest import shutil import markdown -from resource_container.ResourceContainer import RC -from client.preprocessors import do_preprocess, TaPreprocessor -from general_tools.file_utils import unzip, read_file +from libraries.resource_container.ResourceContainer import RC +from libraries.client.preprocessors import do_preprocess, TaPreprocessor +from libraries.general_tools.file_utils import unzip, read_file from bs4 import BeautifulSoup diff --git a/tests/converter_tests/test_md2html_converter.py b/tests/converter_tests/test_md2html_converter.py index b27f66ce..5b3b6cc7 100644 --- a/tests/converter_tests/test_md2html_converter.py +++ b/tests/converter_tests/test_md2html_converter.py @@ -4,8 +4,8 @@ import unittest import codecs from contextlib import closing -from converters.md2html_converter import Md2HtmlConverter -from general_tools.file_utils import remove_tree, unzip, remove +from libraries.converters.md2html_converter import Md2HtmlConverter +from libraries.general_tools.file_utils import remove_tree, unzip, remove from bs4 import BeautifulSoup diff --git a/tests/converter_tests/test_usfm2html_converter.py b/tests/converter_tests/test_usfm2html_converter.py index dedde627..1be1364b 100644 --- a/tests/converter_tests/test_usfm2html_converter.py +++ b/tests/converter_tests/test_usfm2html_converter.py @@ -4,8 +4,8 @@ import tempfile import unittest from contextlib import closing -from converters.usfm2html_converter import Usfm2HtmlConverter -from general_tools.file_utils import remove_tree, unzip, remove +from libraries.converters.usfm2html_converter import Usfm2HtmlConverter +from libraries.general_tools.file_utils import remove_tree, unzip, remove class TestUsfmHtmlConverter(unittest.TestCase): diff --git a/tests/door43_tools_tests/test_obs_handler.py b/tests/door43_tools_tests/test_obs_handler.py index 760b0a04..580d60aa 100644 --- a/tests/door43_tools_tests/test_obs_handler.py +++ b/tests/door43_tools_tests/test_obs_handler.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, unicode_literals, print_function import os import unittest -from door43_tools.obs_handler import OBSInspection +from libraries.door43_tools.obs_handler import OBSInspection class TestObsHandler(unittest.TestCase): diff --git a/tests/door43_tools_tests/test_project_deployer.py b/tests/door43_tools_tests/test_project_deployer.py index 4b2f27a9..4f3906f8 100644 --- a/tests/door43_tools_tests/test_project_deployer.py +++ b/tests/door43_tools_tests/test_project_deployer.py @@ -3,8 +3,8 @@ import os import tempfile from moto import mock_s3 -from door43_tools.project_deployer import ProjectDeployer -from general_tools.file_utils import unzip +from libraries.door43_tools.project_deployer import ProjectDeployer +from libraries.general_tools.file_utils import unzip from shutil import rmtree diff --git a/tests/door43_tools_tests/test_project_printer.py b/tests/door43_tools_tests/test_project_printer.py index f344d12b..d9dc0cc9 100644 --- a/tests/door43_tools_tests/test_project_printer.py +++ b/tests/door43_tools_tests/test_project_printer.py @@ -4,8 +4,8 @@ import tempfile from moto import mock_s3 from bs4 import BeautifulSoup -from door43_tools.project_printer import ProjectPrinter -from general_tools.file_utils import unzip +from libraries.door43_tools.project_printer import ProjectPrinter +from libraries.general_tools.file_utils import unzip from shutil import rmtree diff --git a/tests/door43_tools_tests/test_templaters.py b/tests/door43_tools_tests/test_templaters.py index d49c3711..ae0a593b 100644 --- a/tests/door43_tools_tests/test_templaters.py +++ b/tests/door43_tools_tests/test_templaters.py @@ -6,8 +6,8 @@ import shutil import re from bs4 import BeautifulSoup -from door43_tools.templaters import do_template -from general_tools.file_utils import unzip, read_file +from libraries.door43_tools.templaters import do_template +from libraries.general_tools.file_utils import unzip, read_file class TestTemplater(unittest.TestCase): diff --git a/tests/general_tools_tests/test_file_utils.py b/tests/general_tools_tests/test_file_utils.py index b7f8ecc1..f571f41b 100644 --- a/tests/general_tools_tests/test_file_utils.py +++ b/tests/general_tools_tests/test_file_utils.py @@ -6,7 +6,7 @@ import unittest import zipfile -from general_tools import file_utils +from libraries.general_tools import file_utils class FileUtilsTests(unittest.TestCase): diff --git a/tests/general_tools_tests/test_smartquotes.py b/tests/general_tools_tests/test_smartquotes.py index 13ecc3a9..fc09ba36 100644 --- a/tests/general_tools_tests/test_smartquotes.py +++ b/tests/general_tools_tests/test_smartquotes.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals, print_function import unittest -from general_tools import smartquotes +from libraries.general_tools import smartquotes @unittest.skip("smartquotes not working on lambda") diff --git a/tests/general_tools_tests/test_url_utils.py b/tests/general_tools_tests/test_url_utils.py index 88f61506..4096b3db 100644 --- a/tests/general_tools_tests/test_url_utils.py +++ b/tests/general_tools_tests/test_url_utils.py @@ -3,7 +3,7 @@ import tempfile from six import BytesIO import unittest -from general_tools import url_utils +from libraries.general_tools import url_utils class UrlUtilsTests(unittest.TestCase): diff --git a/tests/gogs_tools_tests/test_gogs_utils.py b/tests/gogs_tools_tests/test_gogs_utils.py index 65fe0603..8357581d 100644 --- a/tests/gogs_tools_tests/test_gogs_utils.py +++ b/tests/gogs_tools_tests/test_gogs_utils.py @@ -2,7 +2,7 @@ import mock import unittest -from gogs_tools.gogs_handler import GogsHandler +from libraries.gogs_tools.gogs_handler import GogsHandler class GogsHandlerTests(unittest.TestCase): diff --git a/tests/handler_tests/__init__.py b/tests/handler_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/handler_tests/test_handlers.py b/tests/handler_tests/test_handlers.py new file mode 100644 index 00000000..d24753ac --- /dev/null +++ b/tests/handler_tests/test_handlers.py @@ -0,0 +1,33 @@ +from __future__ import absolute_import, unicode_literals +import functions +from mock import patch +from functions.client_callback.main import handle +from functions.client_webhook.main import handle +from functions.convert_md2html.main import handle +from functions.convert_usfm2html.main import handle +from functions.dashboard.main import handle +from functions.door43_deploy.main import handle +from functions.list_endpoints.main import handle +from functions.register_module.main import handle +from functions.request_job.main import handle +from functions.start_job.main import handle +from functions.door43_print.main import handle +from unittest import TestCase + + +class TestHandlers(TestCase): + + @patch('libraries.lambda_handlers.handler.Handler.handle') + def test_handlers(self, mock_handle_return_value): + mock_handle_return_value.return_value = None + self.assertIsNone(functions.client_callback.main.handle({}, {})) + self.assertIsNone(functions.client_webhook.main.handle({}, {})) + self.assertIsNone(functions.convert_md2html.main.handle({}, {})) + self.assertIsNone(functions.convert_usfm2html.main.handle({}, {})) + self.assertIsNone(functions.dashboard.main.handle({}, {})) + self.assertIsNone(functions.door43_deploy.main.handle({}, {})) + self.assertIsNone(functions.list_endpoints.main.handle({}, {})) + self.assertIsNone(functions.register_module.main.handle({}, {})) + self.assertIsNone(functions.request_job.main.handle({}, {})) + self.assertIsNone(functions.start_job.main.handle({}, {})) + self.assertIsNone(functions.door43_print.main.handle({}, {})) diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration_tests/test_conversion.py b/tests/integration_tests/test_conversion.py new file mode 100644 index 00000000..3f0a81dd --- /dev/null +++ b/tests/integration_tests/test_conversion.py @@ -0,0 +1,661 @@ +# coding=utf-8 +from __future__ import print_function, absolute_import, unicode_literals +import json +import tempfile +import unittest +import os +from unittest import TestCase +import requests +import shutil +import time +from libraries.general_tools import file_utils +from libraries.manager.manager import TxManager +from libraries.general_tools.file_utils import unzip +from libraries.aws_tools.s3_handler import S3Handler +from libraries.client.client_webhook import ClientWebhook +from bs4 import BeautifulSoup + +COMMIT_LENGTH = 40 +USE_WEB_HOOK_LAMBDA = True + +class TestConversions(TestCase): + """ + To test locally, you must set two environment variables: + + set TEST_DEPLOYED to "test_deployed" + set GOGS_USER_TOKEN to tx-manager user token + + Integration test will run on dev unless TRAVIS_BRANCH is set to 'master' (then will run on prod) + """ + + def setUp(self): + branch = os.environ.get("TRAVIS_BRANCH", "develop") # default is testing develop branch (dev) + + destination = "dev-" # default + if branch == "master": + destination = "" # no prefix for production + + self.destination = destination + self.api_url = 'https://{0}api.door43.org'.format(destination) + self.pre_convert_bucket = '{0}tx-webhook-client'.format(destination) + self.gogs_url = 'https://git.door43.org'.format(destination) + self.cdn_bucket = '{0}cdn.door43.org'.format(destination) + self.job_table_name = '{0}tx-job'.format(destination) + self.module_table_name = '{0}tx-module'.format(destination) + self.cdn_url = 'https://{0}cdn.door43.org'.format(destination) + + print("Testing on '" + branch + "' branch, e.g.: " + self.api_url) + + def tearDown(self): + """Runs after each test.""" + # delete temp files + if hasattr(self, 'temp_dir') and os.path.isdir(self.temp_dir): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_ts_mat_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/kpb_mat_text_udb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "41-MAT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + def test_ts_acts0_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/awa_act_text_reg.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + def test_ts_psa_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/ceb_psa_text_ulb_L3.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "19-PSA" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + def test_obs_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/en-obs-rc-0.2.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedChapterCount = 50 + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, "", job, chapterCount=expectedChapterCount, fileExt="md") + + def test_obs_conversion_ts_upload(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/hu_obs_text_obs.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedChapterCount = 49 + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, "", job, chapterCount=expectedChapterCount, fileExt="md") + + def test_usfm_en_jud_bundle_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/en-ulb-jud.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputNames = [ "66-JUD" ] + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputNames, job) + + @unittest.skip("Skipping broken conversion that needs to be fixed - conversion takes too long and times out") + def test_usfm_en_bundle_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/en-ulb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputNames = [ "01-GEN", "02-EXO", "03-LEV", "05-DEU" ] + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputNames, job) + + @unittest.skip("Skipping broken conversion that needs to be fixed - conversion takes too long and times out") + def test_usfm_ru_bundle_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/bible_ru.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputNames = [ + "01-GEN", + "02-EXO", + "03-LEV", + "04-NUM", + "05-DEU", + "06-JOS", + "07-JDG", + "08-RUT", + "09-1SA", + "10-2SA", + "11-1KI", + "12-2KI", + "13-1CH", + "14-2CH", + "15-EZR", + "16-NEH", + "17-EST", + "18-JOB", + "19-PSA", + "20-PRO", + "21-ECC", + "22-SNG", + "23-ISA", + "24-JER", + "25-LAM", + "26-EZK", + "27-DAN", + "28-HOS", + "29-JOL", + "30-AMO", + "31-OBA", + "32-JON", + "33-MIC", + "34-NAM", + "35-HAB", + "36-ZEP", + "37-HAG", + "38-ZEC", + "39-MAL", + "41-MAT", + "42-MRK", + "43-LUK", + "44-JHN", + "45-ACT", + "46-ROM", + "47-1CO", + "48-2CO", + "49-GAL", + "50-EPH", + "51-PHP", + "52-COL", + "53-1TH", + "54-2TH", + "55-1TI", + "56-2TI", + "57-TIT", + "58-PHM", + "59-HEB", + "60-JAS", + "61-1PE", + "62-2PE", + "63-1JN", + "64-2JN", + "65-3JN", + "66-JUD", + "67-REV" + ] + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputNames, job) + + def test_ts_acts1_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/tx-manager-test-data/kan-x-aruvu_act_text_udb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + def test_ts_acts2_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/mohanraj/kn-x-bedar_act_text_udb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + @unittest.skip("Skip test for time reasons - leave for standalone testing") + def test_ts_acts3_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/nirmala/te-x-budugaja_act_text_reg.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + @unittest.skip("Skip test for time reasons - leave for standalone testing") + def test_ts_acts4_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/jathapu/kxv_act_text_udb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + @unittest.skip("Skip test for time reasons - leave for standalone testing") + def test_ts_acts5_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/vinaykumar/kan-x-thigularu_act_text_udb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + @unittest.skip("Skip test for time reasons - leave for standalone testing") + def test_ts_acts6_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/Zipson/yeu_act_text_udb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + @unittest.skip("Skip test for time reasons - leave for standalone testing") + def test_ts_acts7_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/Zipson/kfc_act_text_udb.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + @unittest.skip("Skip test for time reasons - leave for standalone testing") + def test_ts_acts8_conversion(self): + # given + if not self.isTestingEnabled(): return # skip test if integration test not enabled + git_url = "https://git.door43.org/E01877C8393A/uw-act_udb-aen.git" + baseUrl, repo, user = self.getPartsOfGitUrl(git_url) + expectedOutputName = "45-ACT" + + # when + build_log_json, commitID, commitPath, commitSha, success, job = self.doConversionForRepo(baseUrl, user, repo) + + # then + self.validateConversion(user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputName, job) + + ## + ## handlers + ## + + def isTestingEnabled(self): + test = os.environ.get('TEST_DEPLOYED',"") + doTest = (test == "test_deployed") + if not doTest: + print("Skip testing since TEST_DEPLOYED is not set") + else: + gogsUserToken = os.environ.get('GOGS_USER_TOKEN',"") + self.assertTrue(len(gogsUserToken) > 0, "GOGS_USER_TOKEN is missing in environment") + return doTest + + def getPartsOfGitUrl(self, git_url): + print("Testing conversion of: " + git_url) + parts = git_url.split("/") + baseUrl = "/".join(parts[0:3]) + user = parts[3] + repo = parts[4].split(".git")[0] + return baseUrl, repo, user + + def validateConversion(self, user, repo, success, build_log_json, commitID, commitSha, commitPath, expectedOutputNames, job, chapterCount=-1, fileExt=""): + self.assertTrue(len(build_log_json) > 0) + self.assertIsNotNone(job) + self.temp_dir = tempfile.mkdtemp(prefix='testing_') + + if not (type(expectedOutputNames) is list): + expectedOutputNames = [ expectedOutputNames ] # put string in list + + # check pre-convert files + self.downloadAndCheckZipFile(self.s3_handler, expectedOutputNames, "usfm", self.getPreconvertS3Key(commitSha), + "preconvert", success, chapterCount, fileExt) + + # check deployed files + self.checkDestinationFiles(self.cdn_handler, expectedOutputNames, "html", + self.getDestinationS3Key(commitSha, repo, user), chapterCount) + + self.assertEqual(len(commitID), COMMIT_LENGTH) + self.assertIsNotNone(commitSha) + self.assertIsNotNone(commitPath) + if len(job.errors) > 0: + print("WARNING: Found job errors: " + str(job.errors)) + + if len(build_log_json['errors']) > 0: + print("WARNING: Found build_log errors: " + str(build_log_json['errors'])) + + self.assertTrue(success) + + def downloadAndCheckZipFile(self, handler, expectedOutputFiles, extension, key, type, success, chapterCount=-1, fileExt=""): + zipPath = os.path.join(self.temp_dir, type + ".zip") + handler.download_file(key, zipPath) + temp_sub_dir = tempfile.mkdtemp(dir=self.temp_dir, prefix=type + "_") + unzip(zipPath, temp_sub_dir) + + checkList = [] + if chapterCount <= 0: + for file in expectedOutputFiles: + checkList.append(file + "." + extension) + else: + checkList = ['{0:0>2}.{1}'.format(i, fileExt) for i in range(1, chapterCount + 1)] + + isFirst = True + for file in checkList: + outputFilePath = os.path.join(temp_sub_dir, file) + print("checking preconvert zip for: " + outputFilePath) + self.assertTrue(os.path.exists(outputFilePath), "missing file: " + file) + if isFirst: + self.printFile(file, outputFilePath) + isFirst = False + + manifest_json = os.path.join(temp_sub_dir, "manifest.json") + json_exists = os.path.exists(manifest_json) + if not success and json_exists: # print out for troubleshooting + self.printFile("manifest.json", manifest_json) + manifest_yaml = os.path.join(temp_sub_dir, "manifest.yaml") + yaml_exists = os.path.exists(manifest_yaml) + if not success and yaml_exists: # print out for troubleshooting + self.printFile("manifest.yaml", manifest_yaml) + + self.assertTrue(json_exists or yaml_exists, "missing manifest file") + + def printFile(self, fileName, filePath): + text = file_utils.read_file(filePath) + print("Output file (" + fileName + "): " + text) + + def checkDestinationFiles(self, handler, expectedOutputFiles, extension, key, chapterCount=-1): + checkList = [] + if chapterCount <= 0: + for file in expectedOutputFiles: + checkList.append(file + "." + extension) + else: + checkList = ['{0:0>2}.html'.format(i) for i in range(1, chapterCount + 1)] + # checkList.append("index.html") + + retry_count = 0; + for file in checkList: + path = os.path.join(key, file) + print("checking destination folder for: " + path) + output = handler.get_file_contents(path) + while output==None: # try again in a moment since upload files may not be finished + time.sleep(5) + retry_count += 1 + if retry_count > 7: + print("timeout getting file") + break + + print("retry fetch of: " + path) + output = handler.get_file_contents(path) + + self.assertIsNotNone(output, "missing file: " + path) + + manifest = handler.get_file_contents(os.path.join(key, "manifest.json") ) + if manifest == None: + manifest = handler.get_file_contents(os.path.join(key, "manifest.yaml") ) + self.assertTrue(len(manifest) > 0, "missing manifest file ") + + def doConversionForRepo(self, baseUrl, user, repo): + build_log_json = None + job = None + success = False + self.cdn_handler = S3Handler(self.cdn_bucket) + commitID, commitPath, commitSha = self.fetchCommitDataForRepo(baseUrl, repo, user) # TODO: change this to use gogs API when finished + commitLen = len(commitID) + if commitLen == COMMIT_LENGTH: + self.deletePreconvertZipFile(commitSha) + self.deleteTxOutputZipFile(commitID) + self.emptyDestinationFolder(commitSha, repo, user) + build_log_json, success, job = self.doConversionJob(baseUrl, commitID, commitPath, commitSha, repo, user) + + return build_log_json, commitID, commitPath, commitSha, success, job + + def emptyDestinationFolder(self, commitSha, repo, user): + destination_key = self.getDestinationS3Key(commitSha, repo, user) + for obj in self.cdn_handler.get_objects(prefix=destination_key): + print("deleting destination file: " + obj.key) + self.cdn_handler.delete_file(obj.key) + + def deletePreconvertZipFile(self, commitSha): + self.s3_handler = S3Handler(self.pre_convert_bucket) + preconvert_key = self.getPreconvertS3Key(commitSha) + if self.s3_handler.key_exists(preconvert_key): + print("deleting preconvert file: " + preconvert_key) + self.s3_handler.delete_file(preconvert_key, catch_exception=True) + + def deleteTxOutputZipFile(self, commitID): + txOutput_key = self.getTxOutputS3Key(commitID) + if self.cdn_handler.key_exists(txOutput_key): + print("deleting tx output file: " + txOutput_key) + self.cdn_handler.delete_file(txOutput_key, catch_exception=True) + + def getTxOutputS3Key(self, commitID): + output_key = 'tx/job/{0}.zip'.format(commitID) + return output_key + + def getDestinationS3Key(self, commitSha, repo, user): + destination_key = 'u/{0}/{1}/{2}'.format(user, repo, commitSha) + return destination_key + + def getPreconvertS3Key(self, commitSha): + preconvert_key = "preconvert/{0}.zip".format(commitSha) + return preconvert_key + + def doConversionJob(self, baseUrl, commitID, commitPath, commitSha, repo, user): + gogsUserToken = os.environ.get('GOGS_USER_TOKEN',"") + if len(gogsUserToken) == 0: + print("GOGS_USER_TOKEN is missing in environment") + + webhookData = { + "after": commitID, + "commits": [ + { + "id": "b9278437b27024e07d02490400138d4fd7d1677c", + "message": "Fri Dec 16 2016 11:09:07 GMT+0530 (India Standard Time)\n", + "url": baseUrl + commitPath, + }], + "compare_url": "", + "repository": { + "name": repo, + "owner": { + "id": 1234567890, + "username": user, + "full_name": user, + "email": "you@example.com" + }, + }, + "pusher": { + "id": 123456789, + "username": "test", + "full_name": "", + "email": "you@example.com" + }, + } + env_vars = { + 'api_url': self.api_url, + 'pre_convert_bucket': self.pre_convert_bucket, + 'cdn_bucket': self.cdn_bucket, + 'gogs_url': self.gogs_url, + 'gogs_user_token': gogsUserToken, + 'commit_data': webhookData + } + + if USE_WEB_HOOK_LAMBDA: + headers = {"content-type": "application/json"} + tx_client_webhook_url = "{0}/client/webhook".format(self.api_url) + print('Making request to client/webhook URL {0} with payload:'.format(tx_client_webhook_url), end=' ') + print(webhookData) + response = requests.post(tx_client_webhook_url, json=webhookData, headers=headers) + print('webhook finished with code:' + str(response.status_code)) + print('webhook finished with text:' + str(response.text)) + build_log_json = json.loads(response.text) + if response.status_code != 200: + return build_log_json, False, (build_log_json['job_id']) + + else: # do preconvert locally + try: + build_log_json = ClientWebhook(**env_vars).process_webhook() + except Exception as e: + message = "Exception: " + str(e) + print(message) + return None, False, None + + job_id = build_log_json['job_id'] + if job_id == None: + print("Job ID missing in build_log") + return None, False, None + + success, job = self.pollUntilJobFinished(job_id) + build_log_json = self.getJsonFile(commitSha, 'build_log.json', repo, user) + if build_log_json != None: + print("Final results:\n" + str(build_log_json)) + return build_log_json, success, job + + def pollUntilJobFinished(self, job_id): + success = False + job = None + + env_vars = { + 'api_url': self.api_url, + 'gogs_url': self.gogs_url, + 'cdn_url': self.cdn_url, + 'job_table_name': self.job_table_name, + 'module_table_name': self.module_table_name, + 'cdn_bucket': self.cdn_bucket + } + tx_manager = TxManager(**env_vars) + + pollingTimeout = 5 * 60 # poll for up to 5 minutes for job to complete or error + sleepInterval = 5 # how often to check for completion + startMaxWaitCount = 30 / sleepInterval # maximum count to wait for conversion to start (sec/interval) + for i in range(0, pollingTimeout / sleepInterval): + job = tx_manager.get_job(job_id) + self.assertIsNotNone(job) + print("job status at " + str(i) + ":\n" + str(job.log)) + + if job.ended_at != None: + success = True + break + + if (i > startMaxWaitCount) and (job.started_at == None): + success = False + print("Conversion Failed to start") + break + + time.sleep(sleepInterval) # delay before polling again + + return success, job + + def getJsonFile(self, commitSha, file, repo, user): + key = 'u/{0}/{1}/{2}/{3}'.format(user, repo, commitSha, file) + text = self.cdn_handler.get_json(key) + return text + + def fetchCommitDataForRepo(self, baseUrl, repo, user): + commitID = None + commitSha = None + commitPath = None + data = self.readContentsOfRepo(baseUrl, user, repo) + if len(data) > 10: + commitID, commitSha, commitPath = self.findLastedCommitFromPage(data) + return commitID, commitPath, commitSha + + def findLastedCommitFromPage(self, text): + soup = BeautifulSoup(text, 'html.parser') + table = soup.find('table') + commitID = None + commitSha = None + commitPath = None + if table != None: + rows = table.findAll('tr') + if (rows != None) and (len(rows) > 0): + for row in rows: + commitCell = row.find('td', {"class": "sha"} ) + if commitCell != None: + commitLink = commitCell.find('a') + if commitLink != None: + commitPath = commitLink['href'] + commitSha = self.getContents(commitLink) + parts = commitPath.split('/') + commitID = parts[4] + break + + return commitID, commitSha, commitPath + + def makeFolder(self, directory): + if not os.path.exists(directory): + os.makedirs(directory) + + def readContentsOfRepo(self, baseUrl, user, repo): + self.url = "{0}/{1}/{2}/commits/master".format(baseUrl, user, repo) + ttr_response = requests.get(self.url) + if ttr_response.status_code == 200: + return ttr_response.text + + print("Failed to load: " + self.url) + return None + + def getContents(self, item): + if item != None: + contents = item.stripped_strings + for string in contents: + text = string + return text + return None diff --git a/tests/lambda_handlers_tests/test_clientCallbackHandler.py b/tests/lambda_handlers_tests/test_clientCallbackHandler.py index 7bf5fc97..047933f6 100644 --- a/tests/lambda_handlers_tests/test_clientCallbackHandler.py +++ b/tests/lambda_handlers_tests/test_clientCallbackHandler.py @@ -1,12 +1,12 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.client_callback_handler import ClientCallbackHandler +from libraries.lambda_handlers.client_callback_handler import ClientCallbackHandler class TestClientCallbackHandler(TestCase): - @mock.patch('client.client_callback.ClientCallback.process_callback') + @mock.patch('libraries.client.client_callback.ClientCallback.process_callback') def test_handle(self, mock_process_callback): mock_process_callback.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_clientWebhookHandler.py b/tests/lambda_handlers_tests/test_clientWebhookHandler.py index 2ab42e8e..017d445d 100644 --- a/tests/lambda_handlers_tests/test_clientWebhookHandler.py +++ b/tests/lambda_handlers_tests/test_clientWebhookHandler.py @@ -1,12 +1,12 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.client_webhook_handler import ClientWebhookHandler +from libraries.lambda_handlers.client_webhook_handler import ClientWebhookHandler class TestClientWebhookHandler(TestCase): - @mock.patch('client.client_webhook.ClientWebhook.process_webhook') + @mock.patch('libraries.client.client_webhook.ClientWebhook.process_webhook') def test_handle(self, mock_process_webhook): mock_process_webhook.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_convertHandler.py b/tests/lambda_handlers_tests/test_convertHandler.py index b6828033..390f8c72 100644 --- a/tests/lambda_handlers_tests/test_convertHandler.py +++ b/tests/lambda_handlers_tests/test_convertHandler.py @@ -1,13 +1,13 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.convert_handler import ConvertHandler -from converters.md2html_converter import Md2HtmlConverter -from converters.usfm2html_converter import Usfm2HtmlConverter +from libraries.lambda_handlers.convert_handler import ConvertHandler +from libraries.converters.md2html_converter import Md2HtmlConverter +from libraries.converters.usfm2html_converter import Usfm2HtmlConverter class TestConvertHandler(TestCase): - @mock.patch('converters.converter.Converter.run') + @mock.patch('libraries.converters.converter.Converter.run') def test_handle_for_obs(self, mock_convert_run): mock_convert_run.return_value = None event = { @@ -49,7 +49,7 @@ def test_handle_for_obs(self, mock_convert_run): } self.assertIsNone(ConvertHandler(Md2HtmlConverter).handle(event, None)) - @mock.patch('converters.converter.Converter.run') + @mock.patch('libraries.converters.converter.Converter.run') def test_handle_for_usfm(self, mock_convert_run): mock_convert_run.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_dashboardHandler.py b/tests/lambda_handlers_tests/test_dashboardHandler.py index 0df6b7c3..b2155281 100644 --- a/tests/lambda_handlers_tests/test_dashboardHandler.py +++ b/tests/lambda_handlers_tests/test_dashboardHandler.py @@ -1,16 +1,16 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.dashboard_handler import DashboardHandler -from manager.manager import TxManager +from libraries.lambda_handlers.dashboard_handler import DashboardHandler +from libraries.manager.manager import TxManager def new_generate_dashboard( max_failures): return max_failures; # return the parameter for testing class DashboardHandlerTest(TestCase): - @mock.patch('manager.manager.TxManager.setup_resources') - @mock.patch('manager.manager.TxManager.generate_dashboard') + @mock.patch('libraries.manager.manager.TxManager.setup_resources') + @mock.patch('libraries.manager.manager.TxManager.generate_dashboard') def test_handle(self, mock_generate_dashboard, mock_setup_resources): mock_generate_dashboard.side_effect=new_generate_dashboard expectedMaxFailures = TxManager.MAX_FAILURES @@ -30,8 +30,8 @@ def test_handle(self, mock_generate_dashboard, mock_setup_resources): maxFailures = handler.handle(event, expectedMaxFailures) self.assertEqual(maxFailures, expectedMaxFailures) - @mock.patch('manager.manager.TxManager.setup_resources') - @mock.patch('manager.manager.TxManager.generate_dashboard') + @mock.patch('libraries.manager.manager.TxManager.setup_resources') + @mock.patch('libraries.manager.manager.TxManager.generate_dashboard') def test_dashboard_handler_max_two(self, mock_generate_dashboard, mock_setup_resources): mock_generate_dashboard.side_effect=new_generate_dashboard expectedMaxFailures = 2 diff --git a/tests/lambda_handlers_tests/test_door43DeployHandler.py b/tests/lambda_handlers_tests/test_door43DeployHandler.py index bf8e92bd..7b95cc4c 100644 --- a/tests/lambda_handlers_tests/test_door43DeployHandler.py +++ b/tests/lambda_handlers_tests/test_door43DeployHandler.py @@ -1,13 +1,13 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.door43_deploy_handler import Door43DeployHandler +from libraries.lambda_handlers.door43_deploy_handler import Door43DeployHandler class TestDoor43DeployerHandler(TestCase): - @mock.patch('door43_tools.project_deployer.ProjectDeployer.redeploy_all_projects') - @mock.patch('door43_tools.project_deployer.ProjectDeployer.deploy_revision_to_door43') + @mock.patch('libraries.door43_tools.project_deployer.ProjectDeployer.redeploy_all_projects') + @mock.patch('libraries.door43_tools.project_deployer.ProjectDeployer.deploy_revision_to_door43') def test_handle(self, mock_deploy_revision_to_door43, mock_redeploy_all_projects): mock_redeploy_all_projects.return_value = None mock_deploy_revision_to_door43.return_value = None diff --git a/tests/lambda_handlers_tests/test_door43PrintHandler.py b/tests/lambda_handlers_tests/test_door43PrintHandler.py index 32ea5f36..2eaaa1cc 100644 --- a/tests/lambda_handlers_tests/test_door43PrintHandler.py +++ b/tests/lambda_handlers_tests/test_door43PrintHandler.py @@ -1,12 +1,12 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.door43_print_handler import Door43PrintHandler +from libraries.lambda_handlers.door43_print_handler import Door43PrintHandler class TestDoor43PrintHandler(TestCase): - @mock.patch('door43_tools.project_printer.ProjectPrinter.print_project') + @mock.patch('libraries.door43_tools.project_printer.ProjectPrinter.print_project') def test_handle(self, mock_print_project): mock_print_project.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_handler.py b/tests/lambda_handlers_tests/test_handler.py index 6437c434..275a9d3a 100644 --- a/tests/lambda_handlers_tests/test_handler.py +++ b/tests/lambda_handlers_tests/test_handler.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, unicode_literals, print_function from unittest import TestCase from mock import patch -from lambda_handlers.handler import Handler +from libraries.lambda_handlers.handler import Handler class MockHandler(Handler): diff --git a/tests/lambda_handlers_tests/test_listEndpointsHandler.py b/tests/lambda_handlers_tests/test_listEndpointsHandler.py index fec82841..425f1a0a 100644 --- a/tests/lambda_handlers_tests/test_listEndpointsHandler.py +++ b/tests/lambda_handlers_tests/test_listEndpointsHandler.py @@ -1,13 +1,13 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.list_endpoints_handler import ListEndpointsHandler +from libraries.lambda_handlers.list_endpoints_handler import ListEndpointsHandler class TestListEndpointsHandler(TestCase): - @mock.patch('manager.manager.TxManager.setup_resources') - @mock.patch('manager.manager.TxManager.list_endpoints') + @mock.patch('libraries.manager.manager.TxManager.setup_resources') + @mock.patch('libraries.manager.manager.TxManager.list_endpoints') def test_handle(self, mock_list_endpoints, mock_setup_resources): mock_list_endpoints.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_listJobsHandler.py b/tests/lambda_handlers_tests/test_listJobsHandler.py index ef4a4011..a6a633f0 100644 --- a/tests/lambda_handlers_tests/test_listJobsHandler.py +++ b/tests/lambda_handlers_tests/test_listJobsHandler.py @@ -1,13 +1,13 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.list_jobs_handler import ListJobsHandler +from libraries.lambda_handlers.list_jobs_handler import ListJobsHandler class TestListJobsHandler(TestCase): - @mock.patch('manager.manager.TxManager.setup_resources') - @mock.patch('manager.manager.TxManager.list_jobs') + @mock.patch('libraries.manager.manager.TxManager.setup_resources') + @mock.patch('libraries.manager.manager.TxManager.list_jobs') def test_handle(self, mock_list_jobs, mock_setup_resources): mock_list_jobs.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_registerModuleHandler.py b/tests/lambda_handlers_tests/test_registerModuleHandler.py index 77618ca0..fc3dab5e 100644 --- a/tests/lambda_handlers_tests/test_registerModuleHandler.py +++ b/tests/lambda_handlers_tests/test_registerModuleHandler.py @@ -1,13 +1,13 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.register_module_handler import RegisterModuleHandler +from libraries.lambda_handlers.register_module_handler import RegisterModuleHandler class TestRegisterModuleHandler(TestCase): - @mock.patch('manager.manager.TxManager.setup_resources') - @mock.patch('manager.manager.TxManager.register_module') + @mock.patch('libraries.manager.manager.TxManager.setup_resources') + @mock.patch('libraries.manager.manager.TxManager.register_module') def test_handle(self, mock_register_module, mock_setup_resources): mock_register_module.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_requestJobHandler.py b/tests/lambda_handlers_tests/test_requestJobHandler.py index a94d328b..8c743485 100644 --- a/tests/lambda_handlers_tests/test_requestJobHandler.py +++ b/tests/lambda_handlers_tests/test_requestJobHandler.py @@ -1,13 +1,13 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.request_job_handler import RequestJobHandler +from libraries.lambda_handlers.request_job_handler import RequestJobHandler class TestRequestJobHandler(TestCase): - @mock.patch('manager.manager.TxManager.setup_resources') - @mock.patch('manager.manager.TxManager.setup_job') + @mock.patch('libraries.manager.manager.TxManager.setup_resources') + @mock.patch('libraries.manager.manager.TxManager.setup_job') def test_handle(self, mock_setup_job, mock_setup_resources): mock_setup_job.return_value = None event = { diff --git a/tests/lambda_handlers_tests/test_startJobHandler.py b/tests/lambda_handlers_tests/test_startJobHandler.py index 4b0e170a..90c0f300 100644 --- a/tests/lambda_handlers_tests/test_startJobHandler.py +++ b/tests/lambda_handlers_tests/test_startJobHandler.py @@ -1,13 +1,13 @@ from __future__ import absolute_import, unicode_literals, print_function import mock from unittest import TestCase -from lambda_handlers.start_job_handler import StartJobHandler +from libraries.lambda_handlers.start_job_handler import StartJobHandler class TestStartJobHandler(TestCase): - @mock.patch('manager.manager.TxManager.setup_resources') - @mock.patch('manager.manager.TxManager.start_job') + @mock.patch('libraries.manager.manager.TxManager.setup_resources') + @mock.patch('libraries.manager.manager.TxManager.start_job') def test_handle(self, mock_start_job, mock_setup_resources): mock_start_job.return_value = None event = { diff --git a/tests/manager_tests/test_manager.py b/tests/manager_tests/test_manager.py index 33929560..b47a0e03 100644 --- a/tests/manager_tests/test_manager.py +++ b/tests/manager_tests/test_manager.py @@ -2,13 +2,12 @@ import itertools import unittest import mock -from six import StringIO from bs4 import BeautifulSoup from tests.manager_tests import mock_utils from tests.manager_tests.mock_utils import MockResponse -from manager.job import TxJob -from manager.manager import TxManager -from manager.module import TxModule +from libraries.manager.job import TxJob +from libraries.manager.manager import TxManager +from libraries.manager.module import TxModule class ManagerTest(unittest.TestCase): @@ -187,8 +186,8 @@ def setUpClass(cls): return_value=mock_utils.mock_gogs_handler(["token1", "token2"])) ManagerTest.patches = ( - mock.patch("manager.manager.DynamoDBHandler", cls.mock_db), - mock.patch("manager.manager.GogsHandler", cls.mock_gogs), + mock.patch("libraries.manager.manager.DynamoDBHandler", cls.mock_db), + mock.patch("libraries.manager.manager.GogsHandler", cls.mock_gogs), ) for patch in ManagerTest.patches: diff --git a/tests/manager_tests/test_object.py b/tests/manager_tests/test_object.py index 490f7f6c..24bfb107 100644 --- a/tests/manager_tests/test_object.py +++ b/tests/manager_tests/test_object.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, unicode_literals, print_function import unittest -from manager.module import TxObject +from libraries.manager.module import TxObject class MyObject(TxObject): diff --git a/tests/resource_container_tests/test_resource_container.py b/tests/resource_container_tests/test_resource_container.py index d11c267b..65b973bc 100644 --- a/tests/resource_container_tests/test_resource_container.py +++ b/tests/resource_container_tests/test_resource_container.py @@ -3,8 +3,8 @@ import tempfile import unittest import mock -from resource_container.ResourceContainer import RC, manifest_from_repo_name -from general_tools.file_utils import remove_tree, unzip, load_yaml_object, load_json_object +from libraries.resource_container.ResourceContainer import RC, manifest_from_repo_name +from libraries.general_tools.file_utils import remove_tree, unzip, load_yaml_object, load_json_object from datetime import datetime @@ -159,7 +159,7 @@ def test_ceb_psa_text_ulb_L3(self): chunks = rc.projects[0].chunks('01') self.assertEqual(len(chunks), 5) - @mock.patch('general_tools.url_utils.get_url') + @mock.patch('libraries.general_tools.url_utils.get_url') def test_random_tests(self, mock_get_url): mock_get_url.return_value = '[{"ld": "ltr", "gw": false, "lc": "aa", "ln": "Afaraf", "cc": ["DJ", "ER", "ET", "US", "CA"], "pk": 6, "alt": ["Afaraf", "Danakil", "Denkel", "Adal", "Afar Af", "Qafar", "Baadu (Ba\'adu)"], "lr": "Africa", "ang": "Afar"}, {"ld": "ltr", "gw": false, "lc": "aaa", "ln": "Ghotuo", "cc": ["NG"], "pk": 7, "alt": [], "lr": "Africa", "ang": "Ghotuo"}]' # Test getting language from tD diff --git a/travis-install-apex.sh b/travis-install-apex.sh new file mode 100755 index 00000000..b1b4593c --- /dev/null +++ b/travis-install-apex.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PARENT_DIR="$( dirname "${THIS_DIR}" )" + +LATEST=$(curl -s https://api.github.com/repos/apex/apex/tags | grep -Eo '"name":.*?[^\\]",' | head -n 1 | sed 's/[," ]//g' | cut -d ':' -f 2) +if [ -z $LATEST ]; then + LATEST=v0.14.0 +fi +URL="https://github.com/apex/apex/releases/download/${LATEST}/apex_linux_amd64" +DEST="${PARENT_DIR}/apex" + +curl -sL ${URL} -o ${DEST} +chmod +x ${DEST} + +echo THIS_DIR=$THIS_DIR +echo PARENT_DIR=$PARENT_DIR +echo LATEST=$LATEST +echo URL=$URL +echo DEST=$DEST +ls -l ${DEST}