Merge pull request #757 from VWS-Python/dockerfiles

Add Dockerfiles to run the Flask application

Author: adamtheturtle

.github/workflows/ci.yml

@@ -64,6 +64,7 @@ jobs:
           - test_update_target.py::TestWidth
           - test_update_target.py::TestInactiveProject
           - test_usage.py
+          - test_docker.py
 
     steps:
       # We share Vuforia credentials and therefore Vuforia databases across

dev-requirements.txt

@@ -7,6 +7,7 @@ autoflake==1.4
 black==20.8b1
 check-manifest==0.43
 doc8==0.8.1
+docker==4.3.1
 dodgy==0.2.1  # Look for uploaded secrets
 flake8-commas==2.0.0  # Require silicon valley commas
 flake8-quotes==3.2.0  # Require single quotes

docs/source/differences-to-vws.rst

@@ -101,3 +101,8 @@ These are:
 * ``TargetQuotaReached``
 * ``ProjectSuspended``
 * ``ProjectHasNoAPIAccess``
+
+``Content-Length`` headers
+--------------------------
+
+When the given ``Content-Length`` header does not match the length of the given data, the mock server (written with Flask) will not behave as the real Vuforia Web Services behaves.

setup.cfg

@@ -19,6 +19,10 @@ ignore =
     CONTRIBUTING.rst
     LICENSE
     Makefile
+    src/mock_vws/_flask_server/dockerfiles/base/Dockerfile
+    src/mock_vws/_flask_server/dockerfiles/storage/Dockerfile
+    src/mock_vws/_flask_server/dockerfiles/vwq/Dockerfile
+    src/mock_vws/_flask_server/dockerfiles/vws/Dockerfile
     ci
     ci/**
     codecov.yaml

setup.py

@@ -31,7 +31,10 @@ def _get_dependencies(requirements_file: Path) -> List[str]:
 )
 
 setup(
-    use_scm_version=True,
+    # We use a dictionary with a fallback version rather than "True"
+    # like https://github.com/pypa/setuptools_scm/issues/77 so that we do not
+    # error in Docker.
+    use_scm_version={'fallback_version': 'FALLBACK_VERSION'},
     setup_requires=SETUP_REQUIRES,
     install_requires=INSTALL_REQUIRES,
     extras_require={'dev': DEV_REQUIRES},

src/mock_vws/_flask_server/dockerfiles/base/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.8-slim-buster
+COPY . /app
+WORKDIR /app
+RUN pip install .
+EXPOSE 5000
+ENTRYPOINT ["python"]

src/mock_vws/_flask_server/dockerfiles/storage/Dockerfile

@@ -0,0 +1,2 @@
+FROM vws-mock:base
+CMD ["src/mock_vws/_flask_server/storage.py"]

src/mock_vws/_flask_server/dockerfiles/vwq/Dockerfile

@@ -0,0 +1,2 @@
+FROM vws-mock:base
+CMD ["src/mock_vws/_flask_server/vwq.py"]

src/mock_vws/_flask_server/dockerfiles/vws/Dockerfile

@@ -0,0 +1,2 @@
+FROM vws-mock:base
+CMD ["src/mock_vws/_flask_server/vws.py"]

src/mock_vws/_flask_server/vwq.py

@@ -5,7 +5,6 @@
 https://library.vuforia.com/articles/Solution/How-To-Perform-an-Image-Recognition-Query
 """
 
-import copy
 import email.utils
 from http import HTTPStatus
 from typing import Final, Set
@@ -27,7 +26,7 @@
 
 CLOUDRECO_FLASK_APP = Flask(import_name=__name__)
 CLOUDRECO_FLASK_APP.config['PROPAGATE_EXCEPTIONS'] = True
-STORAGE_BASE_URL: Final[str] = 'http://todo.com'
+STORAGE_BASE_URL: Final[str] = 'http://vws-mock-storage:5000'
 
 
 def get_all_databases() -> Set[VuforiaDatabase]:
@@ -42,21 +41,25 @@ def get_all_databases() -> Set[VuforiaDatabase]:
 
 
 @CLOUDRECO_FLASK_APP.before_request
-def validate_request() -> None:
+def set_terminate_wsgi_input() -> None:
     """
-    Run validators on the request.
+    We set ``wsgi.input_terminated`` to ``True`` when going through
+    ``requests``, so that requests have the given ``Content-Length`` headers
+    and the given data in ``request.headers`` and ``request.data``.
+
+    We set this to ``False`` when running an application as standalone.
+    This is because when running the Flask application, if this is set,
+    reading ``request.data`` hangs.
+
+    Therefore, when running the real Flask application, the behavior is not the
+    same as the real Vuforia.
+    This is documented as a difference in the documentation for this package.
     """
-    request.environ['wsgi.input_terminated'] = True
-    input_stream_copy = copy.copy(request.input_stream)
-    request_body = input_stream_copy.read()
-    databases = get_all_databases()
-    run_query_validators(
-        request_headers=dict(request.headers),
-        request_body=request_body,
-        request_method=request.method,
-        request_path=request.path,
-        databases=databases,
+    terminate_wsgi_input = CLOUDRECO_FLASK_APP.config.get(
+        'TERMINATE_WSGI_INPUT',
+        False,
     )
+    request.environ['wsgi.input_terminated'] = terminate_wsgi_input
 
 
 class ResponseNoContentTypeAdded(Response):
@@ -94,9 +97,16 @@ def query() -> Response:
     """
     query_processes_deletion_seconds = 0.2
     query_recognizes_deletion_seconds = 0.2
+
     databases = get_all_databases()
-    input_stream_copy = copy.copy(request.input_stream)
-    request_body = input_stream_copy.read()
+    request_body = request.stream.read()
+    run_query_validators(
+        request_headers=dict(request.headers),
+        request_body=request_body,
+        request_method=request.method,
+        request_path=request.path,
+        databases=databases,
+    )
     date = email.utils.formatdate(None, localtime=False, usegmt=True)
 
     try:

src/mock_vws/_flask_server/vws.py

@@ -30,7 +30,7 @@
 
 VWS_FLASK_APP = Flask(import_name=__name__)
 VWS_FLASK_APP.config['PROPAGATE_EXCEPTIONS'] = True
-STORAGE_BASE_URL: Final[str] = 'http://todo.com'
+STORAGE_BASE_URL: Final[str] = 'http://vws-mock-storage:5000'
 
 
 def get_all_databases() -> Set[VuforiaDatabase]:
@@ -60,12 +60,33 @@ class ResponseNoContentTypeAdded(Response):
 VWS_FLASK_APP.response_class = ResponseNoContentTypeAdded
 
 
+@VWS_FLASK_APP.before_request
+def set_terminate_wsgi_input() -> None:
+    """
+    We set ``wsgi.input_terminated`` to ``True`` when going through
+    ``requests``, so that requests have the given ``Content-Length`` headers
+    and the given data in ``request.headers`` and ``request.data``.
+
+    We set this to ``False`` when running an application as standalone.
+    This is because when running the Flask application, if this is set,
+    reading ``request.data`` hangs.
+
+    Therefore, when running the real Flask application, the behavior is not the
+    same as the real Vuforia.
+    This is documented as a difference in the documentation for this package.
+    """
+    terminate_wsgi_input = VWS_FLASK_APP.config.get(
+        'TERMINATE_WSGI_INPUT',
+        False,
+    )
+    request.environ['wsgi.input_terminated'] = terminate_wsgi_input
+
+
 @VWS_FLASK_APP.before_request
 def validate_request() -> None:
     """
     Run validators on the request.
     """
-    request.environ['wsgi.input_terminated'] = True
     databases = get_all_databases()
     run_services_validators(
         request_headers=dict(request.headers),

tests/mock_vws/fixtures/vuforia_backends.py

@@ -93,6 +93,21 @@ def _enable_use_docker_in_memory(
     working_database: VuforiaDatabase,
     inactive_database: VuforiaDatabase,
 ) -> Generator:
+    # We set ``wsgi.input_terminated`` to ``True`` so that when going through
+    # ``requests``, the Flask applications
+    # have the given ``Content-Length`` headers and the given data in
+    # ``request.headers`` and ``request.data``.
+    #
+    # We do not set these in the Flask application itself.
+    # This is because when running the Flask application, if this is set,
+    # reading ``request.data`` hangs.
+    #
+    # Therefore, when running the real Flask application, the behavior is not
+    # the same as the real Vuforia.
+    # This is documented as a difference in the documentation for this package.
+    VWS_FLASK_APP.config['TERMINATE_WSGI_INPUT'] = True
+    CLOUDRECO_FLASK_APP.config['TERMINATE_WSGI_INPUT'] = True
+
     with requests_mock.Mocker(real_http=False) as mock:
         add_flask_app_to_mock(
             mock_obj=mock,

tests/mock_vws/test_docker.py

@@ -0,0 +1,156 @@
+"""
+Tests for running the mock server in Docker.
+"""
+
+import io
+import uuid
+from http import HTTPStatus
+from pathlib import Path
+from typing import Iterator
+
+import docker
+import pytest
+import requests
+from docker.models.networks import Network
+from vws import VWS, CloudRecoService
+
+from mock_vws.database import VuforiaDatabase
+
+
+@pytest.fixture(name='custom_bridge_network')
+def fixture_custom_bridge_network() -> Iterator[Network]:
+    """
+    Yield a custom bridge network which containers can connect to.
+    """
+    client = docker.from_env()
+    network = client.networks.create(
+        name='test-vws-bridge-' + uuid.uuid4().hex,
+        driver='bridge',
+    )
+    try:
+        yield network
+    finally:
+        network.remove()
+
+
+def test_build_and_run(
+    high_quality_image: io.BytesIO,
+    custom_bridge_network: Network,
+) -> None:
+    """
+    It is possible to build Docker images which combine to make a working mock
+    application.
+    """
+    repository_root = Path(__file__).parent.parent.parent
+    client = docker.from_env()
+
+    dockerfile_dir = repository_root / 'src/mock_vws/_flask_server/dockerfiles'
+    base_dockerfile = dockerfile_dir / 'base' / 'Dockerfile'
+    storage_dockerfile = dockerfile_dir / 'storage' / 'Dockerfile'
+    vws_dockerfile = dockerfile_dir / 'vws' / 'Dockerfile'
+    vwq_dockerfile = dockerfile_dir / 'vwq' / 'Dockerfile'
+
+    random = uuid.uuid4().hex
+    base_tag = 'vws-mock:base'
+    storage_tag = 'vws-mock-storage:latest-' + random
+    vws_tag = 'vws-mock-vws:latest-' + random
+    vwq_tag = 'vws-mock-vwq:latest-' + random
+
+    client.images.build(
+        path=str(repository_root),
+        dockerfile=str(base_dockerfile),
+        tag=base_tag,
+    )
+
+    storage_image, _ = client.images.build(
+        path=str(repository_root),
+        dockerfile=str(storage_dockerfile),
+        tag=storage_tag,
+    )
+    vws_image, _ = client.images.build(
+        path=str(repository_root),
+        dockerfile=str(vws_dockerfile),
+        tag=vws_tag,
+    )
+    vwq_image, _ = client.images.build(
+        path=str(repository_root),
+        dockerfile=str(vwq_dockerfile),
+        tag=vwq_tag,
+    )
+
+    database = VuforiaDatabase()
+    storage_container_name = 'vws-mock-storage'
+
+    storage_container = client.containers.run(
+        image=storage_image,
+        detach=True,
+        name=storage_container_name,
+        publish_all_ports=True,
+        network=custom_bridge_network.name,
+    )
+    vws_container = client.containers.run(
+        image=vws_image,
+        detach=True,
+        name='vws-mock-vws-' + random,
+        publish_all_ports=True,
+        network=custom_bridge_network.name,
+    )
+    vwq_container = client.containers.run(
+        image=vwq_image,
+        detach=True,
+        name='vws-mock-vwq-' + random,
+        publish_all_ports=True,
+        network=custom_bridge_network.name,
+    )
+
+    storage_container.reload()
+    storage_port_attrs = storage_container.attrs['NetworkSettings']['Ports']
+    storage_host_ip = storage_port_attrs['5000/tcp'][0]['HostIp']
+    storage_host_port = storage_port_attrs['5000/tcp'][0]['HostPort']
+
+    vws_container.reload()
+    vws_port_attrs = vws_container.attrs['NetworkSettings']['Ports']
+    vws_host_ip = vws_port_attrs['5000/tcp'][0]['HostIp']
+    vws_host_port = vws_port_attrs['5000/tcp'][0]['HostPort']
+
+    vwq_container.reload()
+    vwq_port_attrs = vwq_container.attrs['NetworkSettings']['Ports']
+    vwq_host_ip = vwq_port_attrs['5000/tcp'][0]['HostIp']
+    vwq_host_port = vwq_port_attrs['5000/tcp'][0]['HostPort']
+
+    response = requests.post(
+        url=f'http://{storage_host_ip}:{storage_host_port}/databases',
+        json=database.to_dict(),
+    )
+
+    assert response.status_code == HTTPStatus.CREATED
+
+    vws_client = VWS(
+        server_access_key=database.server_access_key,
+        server_secret_key=database.server_secret_key,
+        base_vws_url=f'http://{vws_host_ip}:{vws_host_port}',
+    )
+
+    target_id = vws_client.add_target(
+        name='example',
+        width=1,
+        image=high_quality_image,
+        active_flag=True,
+        application_metadata=None,
+    )
+
+    vws_client.wait_for_target_processed(target_id=target_id)
+
+    cloud_reco_client = CloudRecoService(
+        client_access_key=database.client_access_key,
+        client_secret_key=database.client_secret_key,
+        base_vwq_url=f'http://{vwq_host_ip}:{vwq_host_port}',
+    )
+
+    matching_targets = cloud_reco_client.query(image=high_quality_image)
+
+    for container in (storage_container, vws_container, vwq_container):
+        container.stop()
+        container.remove()
+
+    assert matching_targets[0].target_id == target_id