Merge pull request #1089 from xcube-dev/forman-x-fix_importing_server_side_contribs

Allow server-side panel code on S3

Author: TejasMorbagal

CHANGES.md

@@ -38,7 +38,14 @@
   respectively. The functionality is provided by the
   `https://github.com/bcdev/chartlets` Python library.
   A working example can be found in `examples/serve/panels-demo`.
-  
+
+* The xcube test helper module `test.s3test` has been enhanced to support 
+  testing the experimental _server-side panels_ described above:
+  - added new decorator `@s3_test()` for individual tests with `timeout` arg;
+  - added new context manager `s3_test_server()` with `timeout` arg to be used 
+    within tests function bodies;
+  - `S3Test`, `@s3_test()`, and `s3_test_server()` now restore environment 
+    variables modified for the Moto S3 test server.  
 
 ## Changes in 1.7.1
 

test/s3test.py

@@ -2,6 +2,8 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
+import contextlib
+import functools
 import os
 import subprocess
 import sys
@@ -10,6 +12,7 @@
 import urllib
 import urllib.error
 import urllib.request
+from typing import Callable
 
 import moto.server
 
@@ -18,48 +21,158 @@
 MOTOSERVER_PATH = moto.server.__file__
 MOTOSERVER_ARGS = [sys.executable, MOTOSERVER_PATH]
 
+# Mocked AWS environment variables for Moto.
+MOTOSERVER_ENV = {
+    "AWS_ACCESS_KEY_ID": "testing",
+    "AWS_SECRET_ACCESS_KEY": "testing",
+    "AWS_SECURITY_TOKEN": "testing",
+    "AWS_SESSION_TOKEN": "testing",
+    "AWS_DEFAULT_REGION": "us-east-1",
+    "AWS_ENDPOINT_URL_S3": MOTO_SERVER_ENDPOINT_URL,
+}
+
 
 class S3Test(unittest.TestCase):
-    _moto_server = None
+    _stop_moto_server = None
 
     @classmethod
     def setUpClass(cls) -> None:
         super().setUpClass()
-
-        """Mocked AWS Credentials for moto."""
-        os.environ["AWS_ACCESS_KEY_ID"] = "testing"
-        os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
-        os.environ["AWS_SECURITY_TOKEN"] = "testing"
-        os.environ["AWS_SESSION_TOKEN"] = "testing"
-        os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
-
-        cls._moto_server = subprocess.Popen(MOTOSERVER_ARGS)
-        t0 = time.perf_counter()
-        running = False
-        while not running and time.perf_counter() - t0 < 60.0:
-            try:
-                with urllib.request.urlopen(MOTO_SERVER_ENDPOINT_URL, timeout=1.0):
-                    running = True
-                    print(
-                        f"moto_server started after {round(1000 * (time.perf_counter() - t0))} ms"
-                    )
-
-            except urllib.error.URLError as e:
-                pass
-        if not running:
-            raise Exception(
-                f"Failed to start moto server after {round(1000 * (time.perf_counter() - t0))} ms"
-            )
+        cls._stop_moto_server = _start_moto_server()
 
     def setUp(self) -> None:
-        # see https://github.com/spulec/moto/issues/2288
-        urllib.request.urlopen(
-            urllib.request.Request(
-                MOTO_SERVER_ENDPOINT_URL + "/moto-api/reset", method="POST"
-            )
-        )
+        _reset_moto_server()
 
     @classmethod
     def tearDownClass(cls) -> None:
-        cls._moto_server.kill()
+        cls._stop_moto_server()
         super().tearDownClass()
+
+
+def s3_test(timeout: float = 60.0, ping_timeout: float = 1.0):
+    """A decorator to run individual tests with a Moto S3 server.
+
+    The decorated tests receives the Moto Server's endpoint URL.
+
+    Args:
+        timeout:
+            Total time in seconds it may take to start the server.
+            Raises if time is exceeded.
+        ping_timeout:
+            Timeout for individual ping requests trying to access the
+            started moto server.
+            The total number of pings is ``int(timeout / ping_timeout)``.
+    """
+
+    def decorator(test_func):
+        @functools.wraps(test_func)
+        def wrapper(*args, **kwargs):
+            stop_moto_server = _start_moto_server(timeout, ping_timeout)
+            try:
+                return test_func(*args, endpoint_url=MOTO_SERVER_ENDPOINT_URL, **kwargs)
+            finally:
+                stop_moto_server()
+
+        return wrapper
+
+    return decorator
+
+
+@contextlib.contextmanager
+def s3_test_server(timeout: float = 60.0, ping_timeout: float = 1.0) -> str:
+    """A context manager that starts a Moto S3 server for testing.
+
+    Args:
+        timeout:
+            Total time in seconds it may take to start the server.
+            Raises if time is exceeded.
+        ping_timeout:
+            Timeout for individual ping requests trying to access the
+            started moto server.
+            The total number of pings is ``int(timeout / ping_timeout)``.
+
+    Returns:
+        The server's endpoint URL
+
+    Raises:
+        Exception: If the server could not be started or
+            if the service is not available after after
+            *timeout* seconds.
+    """
+    stop_moto_server = _start_moto_server(timeout=timeout, ping_timeout=ping_timeout)
+    try:
+        _reset_moto_server()
+        yield MOTO_SERVER_ENDPOINT_URL
+    finally:
+        stop_moto_server()
+
+
+def _start_moto_server(
+    timeout: float = 60.0, ping_timeout: float = 1.0
+) -> Callable[[], None]:
+    """Start a Moto S3 server for testing.
+
+    Args:
+        timeout:
+            Total time in seconds it may take to start the server.
+            Raises if time is exceeded.
+        ping_timeout:
+            Timeout for individual ping requests trying to access the
+            started moto server.
+            The total number of pings is ``int(timeout / ping_timeout)``.
+
+    Returns:
+        A function that stops the server and restores the environment.
+
+    Raises:
+        Exception: If the server could not be started or
+            if the service is not available after
+            *timeout* seconds.
+    """
+
+    prev_env: dict[str, str | None] = {
+        k: os.environ.get(k) for k, v in MOTOSERVER_ENV.items()
+    }
+    os.environ.update(MOTOSERVER_ENV)
+
+    moto_server = subprocess.Popen(MOTOSERVER_ARGS)
+    t0 = time.perf_counter()
+    running = False
+    while not running and time.perf_counter() - t0 < timeout:
+        try:
+            with urllib.request.urlopen(MOTO_SERVER_ENDPOINT_URL, timeout=ping_timeout):
+                running = True
+                print(
+                    f"moto_server started after"
+                    f" {round(1000 * (time.perf_counter() - t0))} ms"
+                )
+
+        except urllib.error.URLError:
+            pass
+    if not running:
+        raise Exception(
+            f"Failed to start moto server"
+            f" after {round(1000 * (time.perf_counter() - t0))} ms"
+        )
+
+    def stop_moto_server():
+        try:
+            moto_server.kill()
+        finally:
+            # Restore environment variables
+            for k, v in prev_env.items():
+                if v is None:
+                    del os.environ[k]
+                else:
+                    os.environ[k] = v
+
+    return stop_moto_server
+
+
+def _reset_moto_server():
+    # see https://github.com/spulec/moto/issues/2288
+    urllib.request.urlopen(
+        urllib.request.Request(
+            MOTO_SERVER_ENDPOINT_URL + "/moto-api/reset", method="POST"
+        )
+    )

test/webapi/helpers.py

@@ -52,6 +52,27 @@ def get_server(
     Raise:
         AssertionError: if API context object can not be determined
     """
+    server_config = get_server_config(server_config)
+    framework = framework or MockFramework()
+    extension_registry = extension_registry or get_extension_registry()
+    return Server(framework, server_config, extension_registry=extension_registry)
+
+
+def get_server_config(
+    server_config: Optional[Union[str, Mapping[str, Any]]] = None
+) -> dict[str, Any]:
+    """Get a server configuration for testing.
+
+    The given ``server_config`` is normalized into a dictionary.
+    If ``server_config`` is a path, the configuration is loaded and its
+    ``base_dir`` key is set to the parent directory of the configuration file.
+
+    Args:
+        server_config: Optional path or directory. Defaults to "config.yml".
+
+    Returns:
+        A configuration dictionary.
+    """
     server_config = server_config or "config.yml"
     if isinstance(server_config, str):
         config_path = server_config
@@ -67,10 +88,7 @@ def get_server(
             server_config["base_dir"] = base_dir
     else:
         assert isinstance(server_config, collections.abc.Mapping)
-
-    framework = framework or MockFramework()
-    extension_registry = extension_registry or get_extension_registry()
-    return Server(framework, server_config, extension_registry=extension_registry)
+    return server_config
 
 
 def get_api_ctx(

test/webapi/res/config-panels.yml

@@ -1,8 +1,8 @@
 Viewer:
   Augmentation:
-    Path: ""
+    Path: "viewer/extensions"
     Extensions:
-      - viewer_panels.ext
+      - my_ext.ext
 
 DataStores:
   - Identifier: test
@@ -16,18 +16,3 @@ DataStores:
 ServiceProvider:
   ProviderName: "Brockmann Consult GmbH"
   ProviderSite: "https://www.brockmann-consult.de"
-  ServiceContact:
-    IndividualName: "Norman Fomferra"
-    PositionName: "Senior Software Engineer"
-    ContactInfo:
-      Phone:
-        Voice: "+49 4152 889 303"
-        Facsimile: "+49 4152 889 330"
-      Address:
-        DeliveryPoint: "HZG / GITZ"
-        City: "Geesthacht"
-        AdministrativeArea: "Herzogtum Lauenburg"
-        PostalCode: "21502"
-        Country: "Germany"
-        ElectronicMailAddress: "[email protected]"
-

test/webapi/res/viewer_panels/__init__.py test/webapi/res/viewer/extensions/my_ext/__init__.py

@@ -7,7 +7,12 @@
 from typing import Optional, Union, Any
 from collections.abc import Mapping
 
+import fsspec
+from chartlets import ExtensionContext
+
+from test.s3test import s3_test
 from test.webapi.helpers import get_api_ctx
+from test.webapi.helpers import get_server_config
 from xcube.webapi.viewer.context import ViewerContext
 from xcube.webapi.viewer.contrib import Panel
 
@@ -46,15 +51,33 @@ def test_config_path_ok(self):
         ctx2 = get_viewer_ctx(server_config=config)
         self.assertEqual(config_path, ctx2.config_path)
 
-    def test_panels(self):
+    def test_panels_local(self):
         ctx = get_viewer_ctx("config-panels.yml")
-        self.assertIsNotNone(ctx.ext_ctx)
-        self.assertIsInstance(ctx.ext_ctx.extensions, list)
-        self.assertEqual(1, len(ctx.ext_ctx.extensions))
-        self.assertIsInstance(ctx.ext_ctx.contributions, dict)
-        self.assertEqual(1, len(ctx.ext_ctx.contributions))
-        self.assertIn("panels", ctx.ext_ctx.contributions)
-        self.assertIsInstance(ctx.ext_ctx.contributions["panels"], list)
-        self.assertEqual(2, len(ctx.ext_ctx.contributions["panels"]))
-        self.assertIsInstance(ctx.ext_ctx.contributions["panels"][0], Panel)
-        self.assertIsInstance(ctx.ext_ctx.contributions["panels"][1], Panel)
+        self.assert_extensions_ok(ctx.ext_ctx)
+
+    @s3_test()
+    def test_panels_s3(self, endpoint_url: str):
+        server_config = get_server_config("config-panels.yml")
+        bucket_name = "xcube-testing"
+        base_dir = server_config["base_dir"]
+        ext_path = server_config["Viewer"]["Augmentation"]["Path"]
+        # Copy test extension to S3 bucket
+        s3_fs: fsspec.AbstractFileSystem = fsspec.filesystem(
+            "s3", endpoint_url=endpoint_url
+        )
+        s3_fs.put(f"{base_dir}/{ext_path}", f"{bucket_name}/{ext_path}", recursive=True)
+        server_config["base_dir"] = f"s3://{bucket_name}"
+        ctx = get_viewer_ctx(server_config)
+        self.assert_extensions_ok(ctx.ext_ctx)
+
+    def assert_extensions_ok(self, ext_ctx: ExtensionContext | None):
+        self.assertIsNotNone(ext_ctx)
+        self.assertIsInstance(ext_ctx.extensions, list)
+        self.assertEqual(1, len(ext_ctx.extensions))
+        self.assertIsInstance(ext_ctx.contributions, dict)
+        self.assertEqual(1, len(ext_ctx.contributions))
+        self.assertIn("panels", ext_ctx.contributions)
+        self.assertIsInstance(ext_ctx.contributions["panels"], list)
+        self.assertEqual(2, len(ext_ctx.contributions["panels"]))
+        self.assertIsInstance(ext_ctx.contributions["panels"][0], Panel)
+        self.assertIsInstance(ext_ctx.contributions["panels"][1], Panel)