diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 168d6e5..5f23ab1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,7 @@ jobs: env: FFMPEG_BUILD_VER: 20210119-553eb07737 GIFSICLE_VER: 1.92 + PYTEST_ADDOPTS: ${{ matrix.python-version == '2.7' && '--ignore=tests/result_storages/test_thumbor_aws_storage.py' || '' }} container: image: ${{ matrix.python-version == '2.7' && 'python:2.7-buster' || null }} @@ -91,10 +92,12 @@ jobs: path: reports/*.xml - name: Combine coverage + if: matrix.python-version != '2.7' run: tox -e coverage-report - name: Upload coverage uses: codecov/codecov-action@v3 + if: matrix.python-version != '2.7' with: file: ./coverage.xml name: ${{ github.workflow }} diff --git a/pyproject.toml b/pyproject.toml index fed528d..26976fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools==44.0.0"] build-backend = "setuptools.build_meta" diff --git a/src/thumbor_video_engine/__init__.py b/src/thumbor_video_engine/__init__.py index 0b09b43..2b6a7db 100644 --- a/src/thumbor_video_engine/__init__.py +++ b/src/thumbor_video_engine/__init__.py @@ -288,3 +288,11 @@ 'the source image is an animated gif and the request accepts it (via ' 'Accept: video/* header)', 'Video') + + +Config.define( + 'FFMPEG_PRESERVE_AUDIO', + False, + 'If True, thumbor-video-engine will not strip the audio track. Defaults to ' + 'False', + 'Video') diff --git a/src/thumbor_video_engine/result_storages/aws_storage.py b/src/thumbor_video_engine/result_storages/aws_storage.py new file mode 100644 index 0000000..7ae154b --- /dev/null +++ b/src/thumbor_video_engine/result_storages/aws_storage.py @@ -0,0 +1,122 @@ +from datetime import datetime, timezone +from deprecated import deprecated +from hashlib import sha1 +from urllib.parse import unquote +from os.path import join + +from thumbor.config import Config +from thumbor.engines import BaseEngine +from thumbor.result_storages import ResultStorageResult +from thumbor.utils import logger +import thumbor_aws.result_storage +from thumbor_aws.utils import normalize_path +from .base import BaseStorage + +Config.define( + "TC_AWS_RANDOMIZE_KEYS", False, "Randomize S3 bucket keys", "tc_aws Compatibility" +) +Config.define( + "TC_AWS_ROOT_IMAGE_NAME", + "", + "When resizing a URL that ends in a slash, what should the corresponding cache key be?", + "tc_aws Compatibility", +) + + +class Storage(BaseStorage, thumbor_aws.result_storage.Storage): + @property + def prefix(self): + auto_component = self.get_auto_path_component() + if auto_component: + return f"{self.root_path}/{auto_component}".lstrip("/") + else: + return self.root_path.lstrip("/") + + def normalize_path(self, path): + if not self.context.config.THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE: + return normalize_path(self.prefix, path) + + segments = [path.lstrip("/")] + + root_path = self.context.config.TC_AWS_RESULT_STORAGE_ROOT_PATH + + if root_path: + segments.insert(0, root_path.lstrip("/")) + + auto_component = self.get_auto_path_component() + if auto_component: + segments.append(auto_component) + + if self.context.config.TC_AWS_RANDOMIZE_KEYS: + segments.insert(0, self._generate_digest(segments)) + + normalized_path = join(*segments) + if normalized_path.endswith("/"): + normalized_path += self.context.config.TC_AWS_ROOT_IMAGE_NAME + + return unquote(normalized_path) + + def _generate_digest(self, segments): + return sha1(".".join(segments).encode("utf-8")).hexdigest() + + async def put(self, image_bytes: bytes) -> str: + file_abspath = self.normalize_path(self.context.request.url) + logger.debug("[RESULT_STORAGE] putting at %s", file_abspath) + content_type = BaseEngine.get_mimetype(image_bytes) + response = await self.upload( + file_abspath, + image_bytes, + content_type, + self.context.config.AWS_DEFAULT_LOCATION, + ) + logger.info("[RESULT_STORAGE] Image uploaded successfully to %s", file_abspath) + return response + + async def get(self) -> ResultStorageResult: + path = self.context.request.url + file_abspath = self.normalize_path(path) + + logger.debug("[RESULT_STORAGE] getting from %s", file_abspath) + + exists = await self.object_exists(file_abspath) + if not exists: + logger.debug("[RESULT_STORAGE] image not found at %s", file_abspath) + return None + + status, body, last_modified = await self.get_data( + self.bucket_name, file_abspath + ) + + if status != 200 or self._is_expired(last_modified): + logger.debug( + "[RESULT_STORAGE] cached image has expired (status %s)", status + ) + return None + + logger.info( + "[RESULT_STORAGE] Image retrieved successfully at %s.", + file_abspath, + ) + + return ResultStorageResult( + buffer=body, + metadata={ + "LastModified": last_modified.replace(tzinfo=timezone.utc), + "ContentLength": len(body), + "ContentType": BaseEngine.get_mimetype(body), + }, + ) + + @deprecated(version="7.0.0", reason="Use result's last_modified instead") + async def last_updated( # pylint: disable=invalid-overridden-method + self, + ) -> datetime: + path = self.context.request.url + file_abspath = self.normalize_path(path) + logger.debug("[RESULT_STORAGE] getting from %s", file_abspath) + + response = await self.get_object_metadata(file_abspath) + return datetime.strptime( + response["ResponseMetadata"]["HTTPHeaders"]["last-modified"], + "%a, %d %b %Y %H:%M:%S %Z", + ) diff --git a/tests/conftest.py b/tests/conftest.py index 50c9e23..665e347 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,28 +5,41 @@ from thumbor.context import Context, ServerParameters, RequestParameters from thumbor.importer import Importer from thumbor.server import configure_log, get_application + try: from shutil import which except ImportError: from thumbor.utils import which +try: + from tests.mock_aio_server import s3_server, s3_client, session # noqa +except: # noqa + + @pytest.fixture + def s3_server(): + yield "http://does.not.exist" + + @pytest.fixture + def s3_client(): + return None + CURR_DIR = os.path.abspath(os.path.dirname(__file__)) @pytest.fixture def storage_path(): - return os.path.join(CURR_DIR, 'data') + return os.path.join(CURR_DIR, "data") @pytest.fixture def ffmpeg_path(): - return os.getenv('FFMPEG_PATH') or which('ffmpeg') + return os.getenv("FFMPEG_PATH") or which("ffmpeg") @pytest.fixture def mp4_buffer(storage_path): - with open(os.path.join(storage_path, 'hotdog.mp4'), mode='rb') as f: + with open(os.path.join(storage_path, "hotdog.mp4"), mode="rb") as f: return f.read() @@ -34,31 +47,38 @@ def mp4_buffer(storage_path): def config(storage_path, ffmpeg_path): Config.allow_environment_variables() return Config( - SECURITY_KEY='changeme', - LOADER='thumbor.loaders.file_loader', - APP_CLASS='thumbor_video_engine.app.ThumborServiceApp', + SECURITY_KEY="changeme", + LOADER="thumbor.loaders.file_loader", + APP_CLASS="thumbor_video_engine.app.ThumborServiceApp", FILTERS=[], FILE_LOADER_ROOT_PATH=storage_path, FFMPEG_PATH=ffmpeg_path, - FFPROBE_PATH=(os.getenv('FFPROBE_PATH') or which('ffprobe')), - STORAGE='thumbor.storages.no_storage') + FFPROBE_PATH=(os.getenv("FFPROBE_PATH") or which("ffprobe")), + STORAGE="thumbor.storages.no_storage", + ) @pytest.fixture def context(config): - config.ENGINE = 'thumbor_video_engine.engines.video' + config.ENGINE = "thumbor_video_engine.engines.video" importer = Importer(config) importer.import_modules() server = ServerParameters( - None, 'localhost', 'thumbor.conf', None, 'info', config.APP_CLASS, - gifsicle_path=which('gifsicle')) + None, + "localhost", + "thumbor.conf", + None, + "info", + config.APP_CLASS, + gifsicle_path=which("gifsicle"), + ) server.security_key = config.SECURITY_KEY req = RequestParameters() - configure_log(config, 'DEBUG') + configure_log(config, "DEBUG") with Context(server=server, config=config, importer=importer) as context: context.request = req diff --git a/tests/mock_aio_server.py b/tests/mock_aio_server.py new file mode 100644 index 0000000..dcdef61 --- /dev/null +++ b/tests/mock_aio_server.py @@ -0,0 +1,49 @@ +import pytest +import pytest_asyncio +import aiobotocore.session +from aiobotocore.config import AioConfig + +from tests.moto_server import MotoService + + +@pytest_asyncio.fixture +async def s3_server(monkeypatch, event_loop): + monkeypatch.setenv("TEST_SERVER_MODE", "true") + monkeypatch.setenv("AWS_SHARED_CREDENTIALS_FILE", "") + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key") + monkeypatch.setenv("AWS_SESSION_TOKEN", "test-session-token") + async with MotoService("s3", ssl=False) as svc: + yield svc.endpoint_url + + +@pytest.fixture +def session(event_loop): + return aiobotocore.session.AioSession() + + +@pytest_asyncio.fixture +async def s3_client( + session, + s3_server, +): + # This depends on mock_attributes because we may want to test event listeners. + # See the documentation of `mock_attributes` for details. + read_timeout = connect_timeout = 5 + region = "us-east-1" + + async with session.create_client( + "s3", + region_name=region, + config=AioConfig( + region_name=region, + signature_version="s3", + read_timeout=read_timeout, + connect_timeout=connect_timeout, + ), + verify=False, + endpoint_url=s3_server, + aws_secret_access_key="xxx", + aws_access_key_id="xxx", + ) as client: + yield client diff --git a/tests/moto_server.py b/tests/moto_server.py new file mode 100644 index 0000000..dbcf162 --- /dev/null +++ b/tests/moto_server.py @@ -0,0 +1,141 @@ +import asyncio +import functools +import logging +import socket +import threading +import time + +# Third Party +import aiohttp +import moto.server +import werkzeug.serving + +host = "127.0.0.1" + +_CONNECT_TIMEOUT = 10 + + +def get_free_tcp_port(release_socket: bool = False): + sckt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sckt.bind((host, 0)) + addr, port = sckt.getsockname() + if release_socket: + sckt.close() + return port + + return sckt, port + + +class MotoService: + """Will Create MotoService. + Service is ref-counted so there will only be one per process. Real Service will + be returned by `__aenter__`.""" + + _services = dict() # {name: instance} + + def __init__(self, service_name: str, port: int = None, ssl: bool = False): + self._service_name = service_name + + if port: + self._socket = None + self._port = port + else: + self._socket, self._port = get_free_tcp_port() + + self._thread = None + self._logger = logging.getLogger("MotoService") + self._refcount = None + self._ip_address = host + self._server = None + self._ssl_ctx = werkzeug.serving.generate_adhoc_ssl_context() if ssl else None + self._schema = "http" if not self._ssl_ctx else "https" + + @property + def endpoint_url(self): + return f"{self._schema}://{self._ip_address}:{self._port}" + + def __call__(self, func): + async def wrapper(*args, **kwargs): + await self._start() + try: + result = await func(*args, **kwargs) + finally: + await self._stop() + return result + + functools.update_wrapper(wrapper, func) + wrapper.__wrapped__ = func + return wrapper + + async def __aenter__(self): + svc = self._services.get(self._service_name) + if svc is None: + self._services[self._service_name] = self + self._refcount = 1 + await self._start() + return self + else: + svc._refcount += 1 + return svc + + async def __aexit__(self, exc_type, exc_val, exc_tb): + self._refcount -= 1 + + if self._socket: + self._socket.close() + self._socket = None + + if self._refcount == 0: + del self._services[self._service_name] + await self._stop() + + def _server_entry(self): + self._main_app = moto.server.DomainDispatcherApplication( + moto.server.create_backend_app, service=self._service_name + ) + self._main_app.debug = True + + if self._socket: + self._socket.close() # release right before we use it + self._socket = None + + self._server = werkzeug.serving.make_server( + self._ip_address, + self._port, + self._main_app, + True, + ssl_context=self._ssl_ctx, + ) + self._server.serve_forever() + + async def _start(self): + self._thread = threading.Thread(target=self._server_entry, daemon=True) + self._thread.start() + + async with aiohttp.ClientSession() as session: + start = time.time() + + while time.time() - start < 10: + if not self._thread.is_alive(): + break + + try: + # we need to bypass the proxies due to monkeypatches + async with session.get( + self.endpoint_url + "/static", + timeout=_CONNECT_TIMEOUT, + verify_ssl=False, + ): + pass + break + except (asyncio.TimeoutError, aiohttp.ClientConnectionError): + await asyncio.sleep(0.5) + else: + await self._stop() # pytest.fail doesn't call stop_process + raise Exception(f"Can not start service: {self._service_name}") + + async def _stop(self): + if self._server: + self._server.shutdown() + + self._thread.join() diff --git a/tests/result_storages/test_thumbor_aws_storage.py b/tests/result_storages/test_thumbor_aws_storage.py new file mode 100644 index 0000000..11ae6da --- /dev/null +++ b/tests/result_storages/test_thumbor_aws_storage.py @@ -0,0 +1,174 @@ +import pytest +from unittest import mock + +try: + import pytest_asyncio +except ImportError: + pytest_asyncio = type("Fake", (object,), {"fixture": pytest.fixture}) +else: + import asyncio +try: + import thumbor_aws.s3_client +except ImportError: + thumbor_aws = None + +from thumbor.engines import BaseEngine +from thumbor_video_engine.engines.video import Engine as VideoEngine + +import tornado.httpserver +import tornado.httpclient + + +@pytest_asyncio.fixture(autouse=True) +async def io_loop(request): + io_loop = tornado.ioloop.IOLoop.current() + assert io_loop.asyncio_loop is asyncio.get_event_loop() + + def _close(): + io_loop.close(all_fds=True) + + request.addfinalizer(_close) + return io_loop + + +@pytest_asyncio.fixture +async def http_server(_unused_port, app): + server = tornado.httpserver.HTTPServer(app) + server.add_socket(_unused_port[0]) + await asyncio.sleep(0) + yield server + server.stop() + await server.close_all_connections() + + +@pytest_asyncio.fixture +async def http_client(http_server, s3_client): + await s3_client.create_bucket(Bucket="my-bucket") + client = tornado.httpclient.AsyncHTTPClient() + yield client + client.close() + + +@pytest.fixture +def config(config, s3_client, s3_server): + config.RESULT_STORAGE = "thumbor_video_engine.result_storages.aws_storage" + config.APP_CLASS = "thumbor_video_engine.app.ThumborServiceApp" + config.RESULT_STORAGE_STORES_UNSAFE = True + config.AUTO_WEBP = True + config.FFMPEG_GIF_AUTO_H264 = True + config.THUMBOR_AWS_RUN_IN_COMPATIBILITY_MODE = True + config.TC_AWS_RESULT_STORAGE_BUCKET = ( + config.AWS_RESULT_STORAGE_BUCKET_NAME + ) = "my-bucket" + config.TC_AWS_ENDPOINT = s3_server + config.AWS_LOADER_S3_ENDPOINT_URL = s3_server + config.AWS_DEFAULT_LOCATION = s3_server + return config + + +@pytest.mark.skipif(thumbor_aws is None, reason="thumbor_aws unavailable") +@pytest.mark.asyncio +@pytest.mark.parametrize( + "auto_suffix,mime_type", + [ + ("", "image/gif"), + ("/webp", "image/webp"), + ("/mp4", "video/mp4"), + ], +) +async def test_s3_result_storage_save( + mocker, config, http_client, base_url, auto_suffix, mime_type, s3_client +): + mocker.spy(thumbor_aws.s3_client.S3Client, "upload") + response = await http_client.fetch( + "%s/unsafe/hotdog.gif" % base_url, headers={"Accept": mime_type} + ) + + assert response.code == 200 + bucket_key = "unsafe/hotdog.gif%s" % auto_suffix + assert thumbor_aws.s3_client.S3Client.upload.mock_calls == [ + mock.call(mocker.ANY, bucket_key, mocker.ANY, mocker.ANY, mocker.ANY) + ] + assert BaseEngine.get_mimetype(response.body) == mime_type + assert response.headers.get("vary") == "Accept" + + +@pytest.mark.skipif(thumbor_aws is None, reason="thumbor_aws unavailable") +@pytest.mark.asyncio +@pytest.mark.parametrize("auto_gif", (False, True)) +@pytest.mark.parametrize( + "bucket_key,mime_type,accepts", + [ + ("unsafe/hotdog.gif", "image/gif", "*/*"), + ("unsafe/hotdog.png", "image/png", "*/*"), + ("unsafe/hotdog.gif/webp", "image/webp", "image/webp"), + ("unsafe/hotdog.gif/mp4", "video/mp4", "video/*"), + ], +) +async def test_s3_result_storage_load( + mocker, + config, + http_client, + base_url, + auto_gif, + bucket_key, + mime_type, + accepts, + s3_client, + storage_path, +): + config = config + config.AUTO_WEBP = auto_gif + config.FFMPEG_GIF_AUTO_H264 = auto_gif + + if mime_type == "image/gif": + config.FFMPEG_GIF_AUTO_H264 = False + + mocker.spy(VideoEngine, "load") + + if not auto_gif and mime_type != "image/png": + bucket_key = "unsafe/hotdog.gif" + mime_type = "image/gif" + + ext = mime_type.rpartition("/")[-1] + + with open("%s/hotdog.%s" % (storage_path, ext), mode="rb") as f: + im_bytes = f.read() + + await s3_client.put_object( + Bucket="my-bucket", Key=bucket_key, Body=im_bytes, ContentType=mime_type + ) + + req_ext = "png" if mime_type == "image/png" else "gif" + response = await http_client.fetch( + "%s/unsafe/hotdog.%s" % (base_url, req_ext), headers={"Accept": accepts} + ) + + assert response.code == 200 + assert response.headers.get("content-type") == mime_type + assert response.body == im_bytes + if auto_gif: + assert response.headers.get("vary") == "Accept" + else: + assert response.headers.get("vary") is None + assert VideoEngine.load.call_count == 0 + + +@pytest.mark.parametrize( + "accepts,bucket_key", + [ + (None, "35cc347e42ac84494cc01bd09c2f00e0199330fb/unsafe/hotdog.gif"), + ("webp", "7376d420d176b808de02128d747d4baa776e417a/unsafe/hotdog.gif/webp"), + ("video", "5062b049349be02a856b065e5895c5ca1714e1d4/unsafe/hotdog.gif/mp4"), + ], +) +@pytest.mark.skipif(thumbor_aws is None, reason="thumbor_aws unavailable") +def test_normalize_path_thumbor_aws_tc_aws_compat_settings(config, context, accepts, bucket_key): + config.TC_AWS_RANDOMIZE_KEYS = True + path = "unsafe/hotdog.gif" + result_storage = context.modules.result_storage + if accepts == "video": + result_storage.context.request.accepts_video = True + elif accepts == "webp": + result_storage.context.request.accepts_webp = True + assert result_storage.normalize_path(path) == bucket_key diff --git a/tox.ini b/tox.ini index d8b8ed1..117e675 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ python = 3.11: py311 [testenv] +install_command = {envpython} -m pip install -v {opts} {packages} commands = {envpython} -m pip install -e . -v pytest --junitxml={toxinidir}/reports/test-{envname}.xml {posargs:--cov-report term} @@ -23,6 +24,7 @@ setenv = passenv = FFMPEG_PATH FFPROBE_PATH + PYTEST_ADDOPTS deps = coverage pytest @@ -31,15 +33,18 @@ deps = pytest-cov py27: thumbor<7 !py27: thumbor >= 7.0.0 - !py27: git+https://github.com/fdintino/aws.git@9caa87ea2bdb88ec25d98cdae676c2e5b4be6b23#egg=tc_aws + !py27: pytest-asyncio + py37: git+https://github.com/fdintino/aws.git@9caa87ea2bdb88ec25d98cdae676c2e5b4be6b23#egg=tc_aws + py38,py39,py310,py311: thumbor-aws py27: tc_aws<7 boto mirakuru + py27: PyYAML==5.3.1 py27: moto[server] <= 2.1.0 py27: flask-cors<4 !py27: moto[server] - !py27: boto3==1.21.21 - !py27: botocore==1.24.21 + py37: boto3==1.21.21 + py37: botocore==1.24.21 [testenv:coverage-report] skip_install = true