diff --git a/README.md b/README.md index d426fa3..99a6820 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ Building is as follows: # uv run sphinx-build -M html docs/source/ docs/build/ ``` +In case of major restructuring, it may be needed to clean up the contents of `docs/_autosummary` and potentially other rst files in `docs`, +followed by re-running the build. +Manually triggering sphinx-apidoc is unnecessary. + ## Credits Base implementation originally by Pieter Moens , diff --git a/docs/source/_autosummary/obelisk.asynchronous.Obelisk.rst b/docs/source/_autosummary/obelisk.asynchronous.Obelisk.rst new file mode 100644 index 0000000..32b4158 --- /dev/null +++ b/docs/source/_autosummary/obelisk.asynchronous.Obelisk.rst @@ -0,0 +1,41 @@ +obelisk.asynchronous.Obelisk +============================ + +.. currentmodule:: obelisk.asynchronous + +.. autoclass:: Obelisk + :members: + :show-inheritance: + :inherited-members: + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~Obelisk.__init__ + ~Obelisk.fetch_single_chunk + ~Obelisk.http_post + ~Obelisk.query + ~Obelisk.query_time_chunked + ~Obelisk.send + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~Obelisk.grace_period + ~Obelisk.token + ~Obelisk.token_expires + ~Obelisk.retry_strategy + ~Obelisk.kind + ~Obelisk.log + + \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.base.BaseClient.rst b/docs/source/_autosummary/obelisk.asynchronous.base.BaseClient.rst new file mode 100644 index 0000000..21bc265 --- /dev/null +++ b/docs/source/_autosummary/obelisk.asynchronous.base.BaseClient.rst @@ -0,0 +1,37 @@ +obelisk.asynchronous.base.BaseClient +==================================== + +.. currentmodule:: obelisk.asynchronous.base + +.. autoclass:: BaseClient + :members: + :show-inheritance: + :inherited-members: + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~BaseClient.__init__ + ~BaseClient.http_post + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~BaseClient.grace_period + ~BaseClient.token + ~BaseClient.token_expires + ~BaseClient.retry_strategy + ~BaseClient.kind + ~BaseClient.log + + \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.producer.rst b/docs/source/_autosummary/obelisk.asynchronous.base.rst similarity index 51% rename from docs/source/_autosummary/obelisk.sync.producer.rst rename to docs/source/_autosummary/obelisk.asynchronous.base.rst index 336177e..bdf0dc5 100644 --- a/docs/source/_autosummary/obelisk.sync.producer.rst +++ b/docs/source/_autosummary/obelisk.asynchronous.base.rst @@ -1,7 +1,7 @@ -obelisk.sync.producer -===================== +obelisk.asynchronous.base +========================= -.. automodule:: obelisk.sync.producer +.. automodule:: obelisk.asynchronous.base .. rubric:: Classes @@ -10,5 +10,5 @@ obelisk.sync.producer :toctree: :template: custom-class-template.rst - Producer + BaseClient \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.client.Client.rst b/docs/source/_autosummary/obelisk.asynchronous.client.Client.rst deleted file mode 100644 index 95f23a5..0000000 --- a/docs/source/_autosummary/obelisk.asynchronous.client.Client.rst +++ /dev/null @@ -1,37 +0,0 @@ -obelisk.asynchronous.client.Client -================================== - -.. currentmodule:: obelisk.asynchronous.client - -.. autoclass:: Client - :members: - :show-inheritance: - :inherited-members: - - - .. automethod:: __init__ - - - .. rubric:: Methods - - .. autosummary:: - - ~Client.__init__ - ~Client.http_post - - - - - - .. rubric:: Attributes - - .. autosummary:: - - ~Client.grace_period - ~Client.token - ~Client.token_expires - ~Client.retry_strategy - ~Client.kind - ~Client.log - - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.client.Obelisk.rst b/docs/source/_autosummary/obelisk.asynchronous.client.Obelisk.rst new file mode 100644 index 0000000..97ca89f --- /dev/null +++ b/docs/source/_autosummary/obelisk.asynchronous.client.Obelisk.rst @@ -0,0 +1,41 @@ +obelisk.asynchronous.client.Obelisk +=================================== + +.. currentmodule:: obelisk.asynchronous.client + +.. autoclass:: Obelisk + :members: + :show-inheritance: + :inherited-members: + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~Obelisk.__init__ + ~Obelisk.fetch_single_chunk + ~Obelisk.http_post + ~Obelisk.query + ~Obelisk.query_time_chunked + ~Obelisk.send + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~Obelisk.grace_period + ~Obelisk.token + ~Obelisk.token_expires + ~Obelisk.retry_strategy + ~Obelisk.kind + ~Obelisk.log + + \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.client.rst b/docs/source/_autosummary/obelisk.asynchronous.client.rst index ad556c4..97e212f 100644 --- a/docs/source/_autosummary/obelisk.asynchronous.client.rst +++ b/docs/source/_autosummary/obelisk.asynchronous.client.rst @@ -10,5 +10,5 @@ obelisk.asynchronous.client :toctree: :template: custom-class-template.rst - Client + Obelisk \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.consumer.Consumer.rst b/docs/source/_autosummary/obelisk.asynchronous.consumer.Consumer.rst deleted file mode 100644 index 6128483..0000000 --- a/docs/source/_autosummary/obelisk.asynchronous.consumer.Consumer.rst +++ /dev/null @@ -1,40 +0,0 @@ -obelisk.asynchronous.consumer.Consumer -====================================== - -.. currentmodule:: obelisk.asynchronous.consumer - -.. autoclass:: Consumer - :members: - :show-inheritance: - :inherited-members: - - - .. automethod:: __init__ - - - .. rubric:: Methods - - .. autosummary:: - - ~Consumer.__init__ - ~Consumer.http_post - ~Consumer.query - ~Consumer.query_time_chunked - ~Consumer.single_chunk - - - - - - .. rubric:: Attributes - - .. autosummary:: - - ~Consumer.grace_period - ~Consumer.token - ~Consumer.token_expires - ~Consumer.retry_strategy - ~Consumer.kind - ~Consumer.log - - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.consumer.rst b/docs/source/_autosummary/obelisk.asynchronous.consumer.rst deleted file mode 100644 index 1a4159e..0000000 --- a/docs/source/_autosummary/obelisk.asynchronous.consumer.rst +++ /dev/null @@ -1,14 +0,0 @@ -obelisk.asynchronous.consumer -============================= - -.. automodule:: obelisk.asynchronous.consumer - - - .. rubric:: Classes - - .. autosummary:: - :toctree: - :template: custom-class-template.rst - - Consumer - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.consumer_test.rst b/docs/source/_autosummary/obelisk.asynchronous.consumer_test.rst deleted file mode 100644 index d81d359..0000000 --- a/docs/source/_autosummary/obelisk.asynchronous.consumer_test.rst +++ /dev/null @@ -1,13 +0,0 @@ -obelisk.asynchronous.consumer\_test -=================================== - -.. automodule:: obelisk.asynchronous.consumer_test - - - .. rubric:: Functions - - .. autosummary:: - :toctree: - - test_demo_igent - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.consumer_test.test_demo_igent.rst b/docs/source/_autosummary/obelisk.asynchronous.consumer_test.test_demo_igent.rst deleted file mode 100644 index f02fbb0..0000000 --- a/docs/source/_autosummary/obelisk.asynchronous.consumer_test.test_demo_igent.rst +++ /dev/null @@ -1,6 +0,0 @@ -obelisk.asynchronous.consumer\_test.test\_demo\_igent -===================================================== - -.. currentmodule:: obelisk.asynchronous.consumer_test - -.. autofunction:: test_demo_igent \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.producer.Producer.rst b/docs/source/_autosummary/obelisk.asynchronous.producer.Producer.rst deleted file mode 100644 index a9226ad..0000000 --- a/docs/source/_autosummary/obelisk.asynchronous.producer.Producer.rst +++ /dev/null @@ -1,38 +0,0 @@ -obelisk.asynchronous.producer.Producer -====================================== - -.. currentmodule:: obelisk.asynchronous.producer - -.. autoclass:: Producer - :members: - :show-inheritance: - :inherited-members: - - - .. automethod:: __init__ - - - .. rubric:: Methods - - .. autosummary:: - - ~Producer.__init__ - ~Producer.http_post - ~Producer.send - - - - - - .. rubric:: Attributes - - .. autosummary:: - - ~Producer.grace_period - ~Producer.token - ~Producer.token_expires - ~Producer.retry_strategy - ~Producer.kind - ~Producer.log - - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.producer.rst b/docs/source/_autosummary/obelisk.asynchronous.producer.rst deleted file mode 100644 index 2df3161..0000000 --- a/docs/source/_autosummary/obelisk.asynchronous.producer.rst +++ /dev/null @@ -1,14 +0,0 @@ -obelisk.asynchronous.producer -============================= - -.. automodule:: obelisk.asynchronous.producer - - - .. rubric:: Classes - - .. autosummary:: - :toctree: - :template: custom-class-template.rst - - Producer - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.asynchronous.rst b/docs/source/_autosummary/obelisk.asynchronous.rst index 06cd884..ce6d748 100644 --- a/docs/source/_autosummary/obelisk.asynchronous.rst +++ b/docs/source/_autosummary/obelisk.asynchronous.rst @@ -1,17 +1,14 @@ -obelisk.asynchronous +obelisk.asynchronous ==================== .. automodule:: obelisk.asynchronous -.. rubric:: Modules + .. rubric:: Classes -.. autosummary:: - :toctree: - :template: custom-module-template.rst - :recursive: - - client - consumer - consumer_test - producer + .. autosummary:: + :toctree: + :template: custom-class-template.rst + + Obelisk + \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.Obelisk.rst b/docs/source/_autosummary/obelisk.sync.Obelisk.rst new file mode 100644 index 0000000..a27f6dd --- /dev/null +++ b/docs/source/_autosummary/obelisk.sync.Obelisk.rst @@ -0,0 +1,36 @@ +obelisk.sync.Obelisk +==================== + +.. currentmodule:: obelisk.sync + +.. autoclass:: Obelisk + :members: + :show-inheritance: + :inherited-members: + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~Obelisk.__init__ + ~Obelisk.fetch_single_chunk + ~Obelisk.query + ~Obelisk.query_time_chunked + ~Obelisk.send + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~Obelisk.loop + ~Obelisk.async_obelisk + + \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.client.Obelisk.rst b/docs/source/_autosummary/obelisk.sync.client.Obelisk.rst new file mode 100644 index 0000000..1c27844 --- /dev/null +++ b/docs/source/_autosummary/obelisk.sync.client.Obelisk.rst @@ -0,0 +1,36 @@ +obelisk.sync.client.Obelisk +=========================== + +.. currentmodule:: obelisk.sync.client + +.. autoclass:: Obelisk + :members: + :show-inheritance: + :inherited-members: + + + .. automethod:: __init__ + + + .. rubric:: Methods + + .. autosummary:: + + ~Obelisk.__init__ + ~Obelisk.fetch_single_chunk + ~Obelisk.query + ~Obelisk.query_time_chunked + ~Obelisk.send + + + + + + .. rubric:: Attributes + + .. autosummary:: + + ~Obelisk.loop + ~Obelisk.async_obelisk + + \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.consumer.rst b/docs/source/_autosummary/obelisk.sync.client.rst similarity index 54% rename from docs/source/_autosummary/obelisk.sync.consumer.rst rename to docs/source/_autosummary/obelisk.sync.client.rst index c742ea6..44c9b4a 100644 --- a/docs/source/_autosummary/obelisk.sync.consumer.rst +++ b/docs/source/_autosummary/obelisk.sync.client.rst @@ -1,7 +1,7 @@ -obelisk.sync.consumer -===================== +obelisk.sync.client +=================== -.. automodule:: obelisk.sync.consumer +.. automodule:: obelisk.sync.client .. rubric:: Classes @@ -10,5 +10,5 @@ obelisk.sync.consumer :toctree: :template: custom-class-template.rst - Consumer + Obelisk \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.consumer.Consumer.rst b/docs/source/_autosummary/obelisk.sync.consumer.Consumer.rst deleted file mode 100644 index 8394ebd..0000000 --- a/docs/source/_autosummary/obelisk.sync.consumer.Consumer.rst +++ /dev/null @@ -1,35 +0,0 @@ -obelisk.sync.consumer.Consumer -============================== - -.. currentmodule:: obelisk.sync.consumer - -.. autoclass:: Consumer - :members: - :show-inheritance: - :inherited-members: - - - .. automethod:: __init__ - - - .. rubric:: Methods - - .. autosummary:: - - ~Consumer.__init__ - ~Consumer.query - ~Consumer.query_time_chunked - ~Consumer.single_chunk - - - - - - .. rubric:: Attributes - - .. autosummary:: - - ~Consumer.loop - ~Consumer.async_consumer - - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.consumer_test.rst b/docs/source/_autosummary/obelisk.sync.consumer_test.rst deleted file mode 100644 index f8d450d..0000000 --- a/docs/source/_autosummary/obelisk.sync.consumer_test.rst +++ /dev/null @@ -1,14 +0,0 @@ -obelisk.sync.consumer\_test -=========================== - -.. automodule:: obelisk.sync.consumer_test - - - .. rubric:: Functions - - .. autosummary:: - :toctree: - - test_demo_igent - test_two_instances - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.consumer_test.test_demo_igent.rst b/docs/source/_autosummary/obelisk.sync.consumer_test.test_demo_igent.rst deleted file mode 100644 index 2b56e9b..0000000 --- a/docs/source/_autosummary/obelisk.sync.consumer_test.test_demo_igent.rst +++ /dev/null @@ -1,6 +0,0 @@ -obelisk.sync.consumer\_test.test\_demo\_igent -============================================= - -.. currentmodule:: obelisk.sync.consumer_test - -.. autofunction:: test_demo_igent \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.consumer_test.test_two_instances.rst b/docs/source/_autosummary/obelisk.sync.consumer_test.test_two_instances.rst deleted file mode 100644 index 727977a..0000000 --- a/docs/source/_autosummary/obelisk.sync.consumer_test.test_two_instances.rst +++ /dev/null @@ -1,6 +0,0 @@ -obelisk.sync.consumer\_test.test\_two\_instances -================================================ - -.. currentmodule:: obelisk.sync.consumer_test - -.. autofunction:: test_two_instances \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.producer.Producer.rst b/docs/source/_autosummary/obelisk.sync.producer.Producer.rst deleted file mode 100644 index 7db923a..0000000 --- a/docs/source/_autosummary/obelisk.sync.producer.Producer.rst +++ /dev/null @@ -1,33 +0,0 @@ -obelisk.sync.producer.Producer -============================== - -.. currentmodule:: obelisk.sync.producer - -.. autoclass:: Producer - :members: - :show-inheritance: - :inherited-members: - - - .. automethod:: __init__ - - - .. rubric:: Methods - - .. autosummary:: - - ~Producer.__init__ - ~Producer.send - - - - - - .. rubric:: Attributes - - .. autosummary:: - - ~Producer.loop - ~Producer.async_producer - - \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.sync.rst b/docs/source/_autosummary/obelisk.sync.rst index d014992..6ebf6c6 100644 --- a/docs/source/_autosummary/obelisk.sync.rst +++ b/docs/source/_autosummary/obelisk.sync.rst @@ -1,16 +1,14 @@ -obelisk.sync +obelisk.sync ============ .. automodule:: obelisk.sync -.. rubric:: Modules + .. rubric:: Classes -.. autosummary:: - :toctree: - :template: custom-module-template.rst - :recursive: - - consumer - consumer_test - producer + .. autosummary:: + :toctree: + :template: custom-class-template.rst + + Obelisk + \ No newline at end of file diff --git a/docs/source/_autosummary/obelisk.types.IngestMode.rst b/docs/source/_autosummary/obelisk.types.IngestMode.rst index 448d4c1..df9d886 100644 --- a/docs/source/_autosummary/obelisk.types.IngestMode.rst +++ b/docs/source/_autosummary/obelisk.types.IngestMode.rst @@ -12,6 +12,59 @@ obelisk.types.IngestMode .. automethod:: __init__ + .. rubric:: Methods + + .. autosummary:: + + ~IngestMode.encode + ~IngestMode.replace + ~IngestMode.split + ~IngestMode.rsplit + ~IngestMode.join + ~IngestMode.capitalize + ~IngestMode.casefold + ~IngestMode.title + ~IngestMode.center + ~IngestMode.count + ~IngestMode.expandtabs + ~IngestMode.find + ~IngestMode.partition + ~IngestMode.index + ~IngestMode.ljust + ~IngestMode.lower + ~IngestMode.lstrip + ~IngestMode.rfind + ~IngestMode.rindex + ~IngestMode.rjust + ~IngestMode.rstrip + ~IngestMode.rpartition + ~IngestMode.splitlines + ~IngestMode.strip + ~IngestMode.swapcase + ~IngestMode.translate + ~IngestMode.upper + ~IngestMode.startswith + ~IngestMode.endswith + ~IngestMode.removeprefix + ~IngestMode.removesuffix + ~IngestMode.isascii + ~IngestMode.islower + ~IngestMode.isupper + ~IngestMode.istitle + ~IngestMode.isspace + ~IngestMode.isdecimal + ~IngestMode.isdigit + ~IngestMode.isnumeric + ~IngestMode.isalpha + ~IngestMode.isalnum + ~IngestMode.isidentifier + ~IngestMode.isprintable + ~IngestMode.zfill + ~IngestMode.format + ~IngestMode.format_map + ~IngestMode.maketrans + ~IngestMode.__init__ + diff --git a/docs/source/_autosummary/obelisk.types.ObeliskKind.rst b/docs/source/_autosummary/obelisk.types.ObeliskKind.rst index 2df7c80..c3aeca1 100644 --- a/docs/source/_autosummary/obelisk.types.ObeliskKind.rst +++ b/docs/source/_autosummary/obelisk.types.ObeliskKind.rst @@ -12,6 +12,59 @@ obelisk.types.ObeliskKind .. automethod:: __init__ + .. rubric:: Methods + + .. autosummary:: + + ~ObeliskKind.encode + ~ObeliskKind.replace + ~ObeliskKind.split + ~ObeliskKind.rsplit + ~ObeliskKind.join + ~ObeliskKind.capitalize + ~ObeliskKind.casefold + ~ObeliskKind.title + ~ObeliskKind.center + ~ObeliskKind.count + ~ObeliskKind.expandtabs + ~ObeliskKind.find + ~ObeliskKind.partition + ~ObeliskKind.index + ~ObeliskKind.ljust + ~ObeliskKind.lower + ~ObeliskKind.lstrip + ~ObeliskKind.rfind + ~ObeliskKind.rindex + ~ObeliskKind.rjust + ~ObeliskKind.rstrip + ~ObeliskKind.rpartition + ~ObeliskKind.splitlines + ~ObeliskKind.strip + ~ObeliskKind.swapcase + ~ObeliskKind.translate + ~ObeliskKind.upper + ~ObeliskKind.startswith + ~ObeliskKind.endswith + ~ObeliskKind.removeprefix + ~ObeliskKind.removesuffix + ~ObeliskKind.isascii + ~ObeliskKind.islower + ~ObeliskKind.isupper + ~ObeliskKind.istitle + ~ObeliskKind.isspace + ~ObeliskKind.isdecimal + ~ObeliskKind.isdigit + ~ObeliskKind.isnumeric + ~ObeliskKind.isalpha + ~ObeliskKind.isalnum + ~ObeliskKind.isidentifier + ~ObeliskKind.isprintable + ~ObeliskKind.zfill + ~ObeliskKind.format + ~ObeliskKind.format_map + ~ObeliskKind.maketrans + ~ObeliskKind.__init__ + diff --git a/docs/source/_autosummary/obelisk.types.TimestampPrecision.rst b/docs/source/_autosummary/obelisk.types.TimestampPrecision.rst index f165ddf..d8972ca 100644 --- a/docs/source/_autosummary/obelisk.types.TimestampPrecision.rst +++ b/docs/source/_autosummary/obelisk.types.TimestampPrecision.rst @@ -12,6 +12,59 @@ obelisk.types.TimestampPrecision .. automethod:: __init__ + .. rubric:: Methods + + .. autosummary:: + + ~TimestampPrecision.encode + ~TimestampPrecision.replace + ~TimestampPrecision.split + ~TimestampPrecision.rsplit + ~TimestampPrecision.join + ~TimestampPrecision.capitalize + ~TimestampPrecision.casefold + ~TimestampPrecision.title + ~TimestampPrecision.center + ~TimestampPrecision.count + ~TimestampPrecision.expandtabs + ~TimestampPrecision.find + ~TimestampPrecision.partition + ~TimestampPrecision.index + ~TimestampPrecision.ljust + ~TimestampPrecision.lower + ~TimestampPrecision.lstrip + ~TimestampPrecision.rfind + ~TimestampPrecision.rindex + ~TimestampPrecision.rjust + ~TimestampPrecision.rstrip + ~TimestampPrecision.rpartition + ~TimestampPrecision.splitlines + ~TimestampPrecision.strip + ~TimestampPrecision.swapcase + ~TimestampPrecision.translate + ~TimestampPrecision.upper + ~TimestampPrecision.startswith + ~TimestampPrecision.endswith + ~TimestampPrecision.removeprefix + ~TimestampPrecision.removesuffix + ~TimestampPrecision.isascii + ~TimestampPrecision.islower + ~TimestampPrecision.isupper + ~TimestampPrecision.istitle + ~TimestampPrecision.isspace + ~TimestampPrecision.isdecimal + ~TimestampPrecision.isdigit + ~TimestampPrecision.isnumeric + ~TimestampPrecision.isalpha + ~TimestampPrecision.isalnum + ~TimestampPrecision.isidentifier + ~TimestampPrecision.isprintable + ~TimestampPrecision.zfill + ~TimestampPrecision.format + ~TimestampPrecision.format_map + ~TimestampPrecision.maketrans + ~TimestampPrecision.__init__ + diff --git a/docs/source/conf.py b/docs/source/conf.py index 25a54b2..f5e2540 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -31,6 +31,7 @@ 'sphinx.ext.doctest', ] autosummary_generate = True +autosummary_ignore_module_all = False templates_path = ['_templates'] exclude_patterns = [] diff --git a/src/obelisk/__init__.py b/src/obelisk/__init__.py index fc049b3..4d6f978 100644 --- a/src/obelisk/__init__.py +++ b/src/obelisk/__init__.py @@ -4,10 +4,10 @@ each with a synchronous and async API. The PyPi package name is ``obelisk-py``, the Python module is called ``obelisk``. -Your starting point will be one of the Consumer or Producer instances in :mod:`~.sync` or :mod:`~.asynchronous` depending on your preferred API. +Your starting point will be one of the Obelisk instances in :mod:`~.sync` or :mod:`~.asynchronous` depending on your preferred API. -Each of the `sync` or `asynchronous` modules will contain two main classes, those being Producer and Consumer. -Those submit or fetch data respectively. +The Obelisk classes in these modules both implement the same interface, +but the asynchronous implementation returns Coroutines. Error handling -------------- diff --git a/src/obelisk/asynchronous/__init__.py b/src/obelisk/asynchronous/__init__.py index a0f4184..3629ad8 100644 --- a/src/obelisk/asynchronous/__init__.py +++ b/src/obelisk/asynchronous/__init__.py @@ -2,6 +2,8 @@ This module contains the asynchronous API to Obelisk-py. These methods all return a :any:`Coroutine`. -Relevant entrance points are :class:`.producer.Producer` to publish data to Obelisk, -or :class:`.consumer.Consumer` to consume data. +Relevant entrance points are :class:`client.Obelisk`. +It can be imported from the :mod:`.client` module, or directly from this one. """ +__all__= ['Obelisk'] +from .client import Obelisk diff --git a/src/obelisk/asynchronous/base.py b/src/obelisk/asynchronous/base.py new file mode 100644 index 0000000..fed5980 --- /dev/null +++ b/src/obelisk/asynchronous/base.py @@ -0,0 +1,204 @@ +from datetime import datetime, timedelta +import logging +import base64 +from typing import Any, Optional + +import httpx + +from obelisk.exceptions import AuthenticationError +from obelisk.strategies.retry import RetryStrategy, \ + NoRetryStrategy +from obelisk.types import ObeliskKind + + +class BaseClient: + """ + Base class handling Obelisk auth and doing the core HTTP communication. + Only exists in asynchronous variety, as it is not usually directly useful for user code. + """ + + _client: str = "" + _secret: str = "" + + token: Optional[str] = None + """Current authentication token""" + token_expires: Optional[datetime] = None + """Deadline after which token is no longer useable""" + + grace_period: timedelta = timedelta(seconds=10) + """Controls how much before the expiration deadline a token will be refreshed.""" + retry_strategy: RetryStrategy + kind: ObeliskKind + + log: logging.Logger + + _token_url = 'https://obelisk.ilabt.imec.be/api/v3/auth/token' + _root_url = 'https://obelisk.ilabt.imec.be/api/v3' + _metadata_url = 'https://obelisk.ilabt.imec.be/api/v3/catalog/graphql' + _events_url = 'https://obelisk.ilabt.imec.be/api/v3/data/query/events' + _ingest_url = 'https://obelisk.ilabt.imec.be/api/v3/data/ingest' + _streams_url = 'https://obelisk.ilabt.imec.be/api/v3/data/streams' + + def __init__(self, client: str, secret: str, + retry_strategy: RetryStrategy = NoRetryStrategy(), + kind: ObeliskKind = ObeliskKind.CLASSIC) -> None: + self._client = client + self._secret = secret + self.retry_strategy = retry_strategy + self.kind = kind + + self.log = logging.getLogger('obelisk') + + self._token_url = kind.token_url + self._root_url = kind.root_url + self._events_url = kind.query_url + self._ingest_url = kind.ingest_url + self._streams_url = kind.stream_url + + async def _get_token(self): + auth_string = str(base64.b64encode( + f'{self._client}:{self._secret}'.encode('utf-8')), 'utf-8') + headers = { + 'Authorization': f'Basic {auth_string}', + 'Content-Type': ('application/json' + if self.kind.use_json_auth else 'application/x-www-form-urlencoded') + } + payload = { + 'grant_type': 'client_credentials' + } + + async with httpx.AsyncClient() as client: + response = None + last_error = None + retry = self.retry_strategy.make() + while not response or await retry.should_retry(): + try: + request = await client.post( + self._token_url, + json=payload if self.kind.use_json_auth else None, + data=payload if not self.kind.use_json_auth else None, + headers=headers) + + response = request.json() + except Exception as e: + last_error = e + self.log.error(e) + continue + + if response is None and last_error is not None: + raise last_error + + if request.status_code != 200: + if 'error' in response: + self.log.warning(f"Could not authenticate, {response['error']}") + raise AuthenticationError + + self.token = response['access_token'] + self.token_expires = (datetime.now() + + timedelta(seconds=response['expires_in'])) + + async def _verify_token(self): + if (self.token is None + or self.token_expires < (datetime.now() - self.grace_period)): + retry = self.retry_strategy.make() + first = True + while first or await retry.should_retry(): + first = False + try: + await self._get_token() + return + except: + continue + + async def http_post(self, url: str, data: Any = None, + params: Optional[dict] = None) -> httpx.Response: + """ + Send an HTTP POST request to Obelisk, + with proper auth. + + Possibly refreshes the authentication token and performs backoff as per `retry_strategy`. + This method is not of stable latency because of these properties. + + No validation is performed on the input data, + callers are responsible for formatting it in a method Obelisk understands. + """ + + await self._verify_token() + + headers = { + 'Authorization': f'Bearer {self.token}', + 'Content-Type': 'application/json' + } + if params is None: + params = {} + async with httpx.AsyncClient() as client: + response = None + retry = self.retry_strategy.make() + last_error = None + while not response or await retry.should_retry(): + if response is not None: + self.log.debug(f"Retrying, last response: {response.status_code}") + + try: + response = await client.post(url, + json=data, + params={k: v for k, v in params.items() if + v is not None}, + headers=headers) + + if response.status_code // 100 == 2: + return response + except Exception as e: + self.log.error(e) + last_error = e + continue + + if not response and last_error: + raise last_error + return response + + + async def http_get(self, url: str, params: Optional[dict] = None) -> httpx.Response: + """ + Send an HTTP GET request to Obelisk, + with proper auth. + + Possibly refreshes the authentication token and performs backoff as per `retry_strategy`. + This method is not of stable latency because of these properties. + + No validation is performed on the input data, + callers are responsible for formatting it in a method Obelisk understands. + """ + + await self._verify_token() + + headers = { + 'Authorization': f'Bearer {self.token}', + 'Content-Type': 'application/json' + } + if params is None: + params = {} + async with httpx.AsyncClient() as client: + response = None + retry = self.retry_strategy.make() + last_error = None + while not response or await retry.should_retry(): + if response is not None: + self.log.debug(f"Retrying, last response: {response.status_code}") + + try: + response = await client.get(url, + params={k: v for k, v in params.items() if + v is not None}, + headers=headers) + + if response.status_code // 100 == 2: + return response + except Exception as e: + self.log.error(e) + last_error = e + continue + + if not response and last_error: + raise last_error + return response diff --git a/src/obelisk/asynchronous/client.py b/src/obelisk/asynchronous/client.py index 96f71b3..5102f34 100644 --- a/src/obelisk/asynchronous/client.py +++ b/src/obelisk/asynchronous/client.py @@ -1,165 +1,283 @@ +import json from datetime import datetime, timedelta -import logging -import base64 -from typing import Any, Optional +from math import floor +from typing import Generator, List, Literal, Optional import httpx +from pydantic import ValidationError -from obelisk.exceptions import AuthenticationError -from obelisk.strategies.retry import RetryStrategy, \ - NoRetryStrategy -from obelisk.types import ObeliskKind +from obelisk.exceptions import ObeliskError +from obelisk.types import Datapoint, IngestMode, QueryResult, TimestampPrecision +from obelisk.asynchronous.base import BaseClient -class Client: + +class Obelisk(BaseClient): """ - Base class handling Obelisk auth and doing the core HTTP communication. - Only exists in asynchronous variety, as it is not usually directly useful for user code. + Component that contains all the logic to consume data from + the Obelisk API (e.g. historical data, sse). + + Obelisk API Documentation: + https://obelisk.docs.apiary.io/ """ - _client: str = "" - _secret: str = "" - - token: Optional[str] = None - """Current authentication token""" - token_expires: Optional[datetime] = None - """Deadline after which token is no longer useable""" - - grace_period: timedelta = timedelta(seconds=10) - """Controls how much before the expiration deadline a token will be refreshed.""" - retry_strategy: RetryStrategy - kind: ObeliskKind - - log: logging.Logger - - _token_url = 'https://obelisk.ilabt.imec.be/api/v3/auth/token' - _root_url = 'https://obelisk.ilabt.imec.be/api/v3' - _metadata_url = 'https://obelisk.ilabt.imec.be/api/v3/catalog/graphql' - _events_url = 'https://obelisk.ilabt.imec.be/api/v3/data/query/events' - _ingest_url = 'https://obelisk.ilabt.imec.be/api/v3/data/ingest' - _streams_url = 'https://obelisk.ilabt.imec.be/api/v3/data/streams' - - def __init__(self, client: str, secret: str, - retry_strategy: RetryStrategy = NoRetryStrategy(), - kind: ObeliskKind = ObeliskKind.CLASSIC) -> None: - self._client = client - self._secret = secret - self.retry_strategy = retry_strategy - self.kind = kind - - self.log = logging.getLogger('obelisk') - - if self.kind == ObeliskKind.HFS: - self._token_url = 'https://obelisk-hfs.discover.ilabt.imec.be/auth/realms/obelisk-hfs/protocol/openid-connect/token' - self._root_url = 'https://obelisk-hfs.discover.ilabt.imec.be' - self._events_url = 'https://obelisk-hfs.discover.ilabt.imec.be/data/query/events' - self._ingest_url = 'https://obelisk-hfs.discover.ilabt.imec.be/data/ingest' - else: - self._token_url = 'https://obelisk.ilabt.imec.be/api/v3/auth/token' - self._root_url = 'https://obelisk.ilabt.imec.be/api/v3' - self._metadata_url = 'https://obelisk.ilabt.imec.be/api/v3/catalog/graphql' - self._events_url = 'https://obelisk.ilabt.imec.be/api/v3/data/query/events' - self._ingest_url = 'https://obelisk.ilabt.imec.be/api/v3/data/ingest' - self._streams_url = 'https://obelisk.ilabt.imec.be/api/v3/data/streams' - - async def _get_token(self): - auth_string = str(base64.b64encode( - f'{self._client}:{self._secret}'.encode('utf-8')), 'utf-8') - headers = { - 'Authorization': f'Basic {auth_string}', - 'Content-Type': ('application/x-www-form-urlencoded' - if self.kind == ObeliskKind.HFS else 'application/json') - } + async def fetch_single_chunk( + self, + datasets: List[str], + metrics: Optional[List[str]] = None, + fields: Optional[List[str]] = None, + from_timestamp: Optional[int] = None, + to_timestamp: Optional[int] = None, + order_by: Optional[dict] = None, + filter_: Optional[dict] = None, + limit: Optional[int] = None, + limit_by: Optional[dict] = None, + cursor: Optional[str] = None, + ) -> QueryResult: + """ + Queries one chunk of events from Obelisk for given parameters, + does not handle paging over Cursors. + + Parameters + ---------- + + datasets : List[str] + List of Dataset IDs. + metrics : Optional[List[str]] = None + List of Metric IDs or wildcards (e.g. `*::number`), defaults to all metrics. + fields : Optional[List[str]] = None + List of fields to return in the result set. + Defaults to `[metric, source, value]` + from_timestamp : Optional[int] = None + Limit output to events after (and including) + this UTC millisecond timestamp, if present. + to_timestamp : Optional[int] = None + Limit output to events before (and excluding) + this UTC millisecond timestamp, if present. + order_by : Optional[dict] = None + Specifies the ordering of the output, + defaults to ascending by timestamp. + See Obelisk docs for format. Caller is responsible for validity. + filter_ : Optional[dict] = None + Limit output to events matching the specified Filter expression. + See Obelisk docs, caller is responsible for validity. + limit : Optional[int] = None + Limit output to a maximum number of events. + Also determines the page size. + Default is server-determined, usually 2500. + limit_by : Optional[dict] = None + Limit the combination of a specific set of Index fields + to a specified maximum number. + cursor : Optional[str] = None + Specifies the next cursor, + used when paging through large result sets. + """ + + # pylint: disable=too-many-arguments + data_range = {"datasets": datasets} + if metrics is not None: + data_range["metrics"] = metrics + payload = { - 'grant_type': 'client_credentials' + "dataRange": data_range, + "cursor": cursor, + "fields": fields, + "from": from_timestamp, + "to": to_timestamp, + "orderBy": order_by, + "filter": filter_, + "limit": limit, + "limitBy": limit_by, } + response = await self.http_post( + self._events_url, data={k: v for k, v in payload.items() if v is not None} + ) + if response.status_code != 200: + self.log.warning(f"Unexpected status code: {response.status_code}") + raise ObeliskError(response.status_code, response.reason_phrase) + + try: + js = response.json() + return QueryResult.model_validate(js) + except json.JSONDecodeError as e: + msg = f"Obelisk response is not a JSON object: {e}" + self.log.warning(msg) + raise ObeliskError(msg) + except ValidationError as e: + msg = f"Response cannot be validated: {e}" + self.log.warning(msg) + raise ObeliskError(msg) + + async def query( + self, + datasets: List[str], + metrics: Optional[List[str]] = None, + fields: Optional[List[str]] = None, + from_timestamp: Optional[int] = None, + to_timestamp: Optional[int] = None, + order_by: Optional[dict] = None, + filter_: Optional[dict] = None, + limit: Optional[int] = None, + limit_by: Optional[dict] = None, + ) -> List[Datapoint]: + """ + Queries data from obelisk, + automatically iterating when a cursor is returned. + + Parameters + ---------- + + datasets : List[str] + List of Dataset IDs. + metrics : Optional[List[str]] = None + List of Metric IDs or wildcards (e.g. `*::number`), defaults to all metrics. + fields : Optional[List[str]] = None + List of fields to return in the result set. + Defaults to `[metric, source, value]` + from_timestamp : Optional[int] = None + Limit output to events after (and including) + this UTC millisecond timestamp, if present. + to_timestamp : Optional[int] = None + Limit output to events before (and excluding) + this UTC millisecond timestamp, if present. + order_by : Optional[dict] = None + Specifies the ordering of the output, + defaults to ascending by timestamp. + See Obelisk docs for format. Caller is responsible for validity. + filter_ : Optional[dict] = None + Limit output to events matching the specified Filter expression. + See Obelisk docs, caller is responsible for validity. + limit : Optional[int] = None + Limit output to a maximum number of events. + Also determines the page size. + Default is server-determined, usually 2500. + limit_by : Optional[dict] = None + Limit the combination of a specific set of Index fields + to a specified maximum number. + """ + + cursor: Optional[str] | Literal[True] = True + result_set: List[Datapoint] = [] + + while cursor: + actual_cursor = cursor if cursor is not True else None + result: QueryResult = await self.fetch_single_chunk( + datasets=datasets, + metrics=metrics, + fields=fields, + from_timestamp=from_timestamp, + to_timestamp=to_timestamp, + order_by=order_by, + filter_=filter_, + limit=limit, + limit_by=limit_by, + cursor=actual_cursor, + ) + result_set.extend(result.items) + cursor = result.cursor + + if limit and len(result_set) >= limit: + """On Obelisk HFS, limit is actually page size, + so continuing to read the cursor will result in a larger than desired + set of results. + + On the other hand, if the limit is very large, + we may need to iterate before we reach the desired limit after all. + """ + break + + return result_set + - async with httpx.AsyncClient() as client: - response = None - last_error = None - retry = self.retry_strategy.make() - while not response or await retry.should_retry(): - try: - request = await client.post( - self._token_url, - json=payload if self.kind == ObeliskKind.CLASSIC else None, - data=payload if self.kind == ObeliskKind.HFS else None, - headers=headers) - - response = request.json() - except Exception as e: - last_error = e - self.log.error(e) - continue - - if response is None and last_error is not None: - raise last_error - - if request.status_code != 200: - if 'error' in response: - self.log.warning(f"Could not authenticate, {response['error']}") - raise AuthenticationError - - self.token = response['access_token'] - self.token_expires = (datetime.now() - + timedelta(seconds=response['expires_in'])) - - async def _verify_token(self): - if (self.token is None - or self.token_expires < (datetime.now() - self.grace_period)): - retry = self.retry_strategy.make() - first = True - while first or await retry.should_retry(): - first = False - try: - await self._get_token() - return - except: - continue - - async def http_post(self, url: str, data: Any = None, - params: Optional[dict] = None) -> httpx.Response: + async def query_time_chunked( + self, + datasets: List[str], + metrics: List[str], + from_time: datetime, + to_time: datetime, + jump: timedelta, + filter_: Optional[dict] = None, + direction: Literal["asc", "desc"] = "asc", + ) -> Generator[List[Datapoint], None, None]: """ - Send an HTTP POST request to Obelisk, - with proper auth. + Fetches all data matching the provided filters, + yielding one chunk at a time. + One "chunk" may require several Obelisk calls to resolve cursors. + By necessity iterates over time, no other ordering is supported. - Possibly refreshes the authentication token and performs backoff as per `retry_strategy`. - This method is not of stable latency because of these properties. + Parameters + ---------- - No validation is performed on the input data, - callers are responsible for formatting it in a method Obelisk understands. + datasets : List[str] + Dataset IDs to query from + metrics : List[str] + IDs of metrics to query + from_time : `datetime.datetime` + Start time to fetch from + to_time : `datetime.datetime` + End time to fetch until. + jump : `datetime.timedelta` + Size of one yielded chunk + filter_ : Optional[dict] = None + Obelisk filter, caller is responsible for correct format + direction : Literal['asc', 'desc'] = 'asc' + Yield older data or newer data first, defaults to older first. """ - await self._verify_token() + current_start = from_time + while current_start < to_time: + yield await self.query( + datasets=datasets, + metrics=metrics, + from_timestamp=floor(current_start.timestamp() * 1000), + to_timestamp=floor((current_start + jump).timestamp() * 1000 - 1), + order_by={"field": ["timestamp"], "ordering": direction}, + filter_=filter_, + ) + current_start += jump - headers = { - 'Authorization': f'Bearer {self.token}', - 'Content-Type': 'application/json' + async def send( + self, + dataset: str, + data: List[dict], + precision: TimestampPrecision = TimestampPrecision.MILLISECONDS, + mode: IngestMode = IngestMode.DEFAULT, + ) -> httpx.Response: + """ + Publishes data to Obelisk + + Parameters + ---------- + dataset : str + ID for the dataset to publish to + data : List[dict] + List of Obelisk-acceptable datapoints. + Exact format varies between Classic or HFS, + caller is responsible for formatting. + precision : :class:`~obelisk.types.TimestampPrecision` = TimestampPrecision.MILLISECONDS + Precision used in the numeric timestamps contained in data. + Ensure it matches to avoid weird errors. + mode : :class:`~obelisk.types.IngestMode` = IngestMode.DEFAULT + See docs for :class:`~obelisk.types.IngestMode`. + + Raises + ------ + + ObeliskError + When the resulting status code is not 204, an empty :exc:`~obelisk.exceptions.ObeliskError` is raised. + """ + + params = { + "datasetId": dataset, + "timestampPrecision": precision.value, + "mode": mode.value, } - if params is None: - params = {} - async with httpx.AsyncClient() as client: - response = None - retry = self.retry_strategy.make() - last_error = None - while not response or await retry.should_retry(): - if response is not None: - self.log.debug(f"Retrying, last response: {response.status_code}") - - try: - response = await client.post(url, - json=data, - params={k: v for k, v in params.items() if - v is not None}, - headers=headers) - - if response.status_code // 100 == 2: - return response - except Exception as e: - self.log.error(e) - last_error = e - continue - - if not response and last_error: - raise last_error - return response + + response = await self.http_post( + f"{self._ingest_url}/{dataset}", data=data, params=params + ) + if response.status_code != 204: + msg = f"An error occured during data ingest. Status {response.status_code}, message: {response.text}" + self.log.warning(msg) + raise ObeliskError(msg) + return response diff --git a/src/obelisk/asynchronous/consumer.py b/src/obelisk/asynchronous/consumer.py deleted file mode 100644 index 878b85c..0000000 --- a/src/obelisk/asynchronous/consumer.py +++ /dev/null @@ -1,218 +0,0 @@ -import json -from datetime import datetime, timedelta -from typing import List, Literal, Generator, Optional - -from pydantic import ValidationError - -from obelisk.asynchronous.client import Client -from obelisk.exceptions import ObeliskError -from obelisk.types import QueryResult, Datapoint - -from math import floor - - -class Consumer(Client): - """ - Component that contains all the logic to consume data from - the Obelisk API (e.g. historical data, sse). - - Obelisk API Documentation: - https://obelisk.docs.apiary.io/ - """ - - async def single_chunk(self, datasets: List[str], metrics: Optional[List[str]] = None, - fields: Optional[List[str]] = None, - from_timestamp: Optional[int] = None, - to_timestamp: Optional[int] = None, - order_by: Optional[dict] = None, - filter_: Optional[dict] = None, - limit: Optional[int] = None, - limit_by: Optional[dict] = None, - cursor: Optional[str] = None) -> QueryResult: - """ - Queries one chunk of events from Obelisk for given parameters, - does not handle paging over Cursors. - - Parameters - ---------- - - datasets : List[str] - List of Dataset IDs. - metrics : Optional[List[str]] = None - List of Metric IDs or wildcards (e.g. `*::number`), defaults to all metrics. - fields : Optional[List[str]] = None - List of fields to return in the result set. - Defaults to `[metric, source, value]` - from_timestamp : Optional[int] = None - Limit output to events after (and including) - this UTC millisecond timestamp, if present. - to_timestamp : Optional[int] = None - Limit output to events before (and excluding) - this UTC millisecond timestamp, if present. - order_by : Optional[dict] = None - Specifies the ordering of the output, - defaults to ascending by timestamp. - See Obelisk docs for format. Caller is responsible for validity. - filter_ : Optional[dict] = None - Limit output to events matching the specified Filter expression. - See Obelisk docs, caller is responsible for validity. - limit : Optional[int] = None - Limit output to a maximum number of events. - Also determines the page size. - Default is server-determined, usually 2500. - limit_by : Optional[dict] = None - Limit the combination of a specific set of Index fields - to a specified maximum number. - cursor : Optional[str] = None - Specifies the next cursor, - used when paging through large result sets. - """ - - # pylint: disable=too-many-arguments - data_range = { - 'datasets': datasets - } - if metrics is not None: - data_range['metrics'] = metrics - - payload = { - 'dataRange': data_range, - 'cursor': cursor, - 'fields': fields, - 'from': from_timestamp, - 'to': to_timestamp, - 'orderBy': order_by, - 'filter': filter_, - 'limit': limit, - 'limitBy': limit_by - } - response = await self.http_post(self._events_url, - data={k: v for k, v in payload.items() if - v is not None}) - if response.status_code != 200: - self.log.warning(f"Unexpected status code: {response.status_code}") - raise ObeliskError(response.status_code, response.reason_phrase) - - try: - js = response.json() - return QueryResult.model_validate(js) - except json.JSONDecodeError as e: - msg = f'Obelisk response is not a JSON object: {e}' - self.log.warning(msg) - raise ObeliskError(msg) - except ValidationError as e: - msg = f"Response cannot be validated: {e}" - self.log.warning(msg) - raise ObeliskError(msg) - - - async def query(self, datasets: List[str], metrics:Optional[List[str]] = None, - fields:Optional[List[str]] = None, - from_timestamp: Optional[int] = None, to_timestamp: Optional[int] = None, - order_by: Optional[dict] = None, - filter_: Optional[dict] = None, - limit: Optional[int] = None, - limit_by: Optional[dict] = None) -> List[Datapoint]: - """ - Queries data from obelisk, - automatically iterating when a cursor is returned. - - Parameters - ---------- - - datasets : List[str] - List of Dataset IDs. - metrics : Optional[List[str]] = None - List of Metric IDs or wildcards (e.g. `*::number`), defaults to all metrics. - fields : Optional[List[str]] = None - List of fields to return in the result set. - Defaults to `[metric, source, value]` - from_timestamp : Optional[int] = None - Limit output to events after (and including) - this UTC millisecond timestamp, if present. - to_timestamp : Optional[int] = None - Limit output to events before (and excluding) - this UTC millisecond timestamp, if present. - order_by : Optional[dict] = None - Specifies the ordering of the output, - defaults to ascending by timestamp. - See Obelisk docs for format. Caller is responsible for validity. - filter_ : Optional[dict] = None - Limit output to events matching the specified Filter expression. - See Obelisk docs, caller is responsible for validity. - limit : Optional[int] = None - Limit output to a maximum number of events. - Also determines the page size. - Default is server-determined, usually 2500. - limit_by : Optional[dict] = None - Limit the combination of a specific set of Index fields - to a specified maximum number. - """ - - cursor: Optional[str] | Literal[True] = True - result_set: List[Datapoint] = [] - - while cursor: - actual_cursor = cursor if cursor is not True else None - result: QueryResult = await self.single_chunk(datasets=datasets, - metrics=metrics, fields=fields, - from_timestamp=from_timestamp, - to_timestamp=to_timestamp, - order_by=order_by, filter_=filter_, - limit=limit, - limit_by=limit_by, - cursor=actual_cursor) - result_set.extend(result.items) - cursor = result.cursor - - if limit and len(result_set) >= limit: - """On Obelisk HFS, limit is actually page size, - so continuing to read the cursor will result in a larger than desired - set of results. - - On the other hand, if the limit is very large, - we may need to iterate before we reach the desired limit after all. - """ - break - - return result_set - - - async def query_time_chunked(self, datasets: List[str], metrics: List[str], - from_time: datetime, to_time: datetime, - jump: timedelta, filter_: Optional[dict] = None, - direction: Literal['asc', 'desc'] = 'asc' - ) -> Generator[List[Datapoint], None, None]: - """ - Fetches all data matching the provided filters, - yielding one chunk at a time. - One "chunk" may require several Obelisk calls to resolve cursors. - By necessity iterates over time, no other ordering is supported. - - Parameters - ---------- - - datasets : List[str] - Dataset IDs to query from - metrics : List[str] - IDs of metrics to query - from_time : `datetime.datetime` - Start time to fetch from - to_time : `datetime.datetime` - End time to fetch until. - jump : `datetime.timedelta` - Size of one yielded chunk - filter_ : Optional[dict] = None - Obelisk filter, caller is responsible for correct format - direction : Literal['asc', 'desc'] = 'asc' - Yield older data or newer data first, defaults to older first. - """ - - current_start = from_time - while current_start < to_time: - yield await self.query(datasets=datasets, metrics=metrics, - from_timestamp=floor(current_start.timestamp() * 1000), - to_timestamp=floor((current_start + jump).timestamp() * 1000 - 1), - order_by={"field": ["timestamp"], "ordering": direction}, - filter_=filter_) - current_start += jump diff --git a/src/obelisk/asynchronous/consumer_test.py b/src/obelisk/asynchronous/consumer_test.py deleted file mode 100644 index 7bd04f2..0000000 --- a/src/obelisk/asynchronous/consumer_test.py +++ /dev/null @@ -1,17 +0,0 @@ -import pytest -from .consumer import Consumer - -pytest_plugins = ('pytest_asyncio',) - -@pytest.mark.asyncio -async def test_demo_igent(): - consumer = Consumer(client="67c716e616c11421cfe2faf6", secret="08dafe89-0389-45b4-9832-cc565fb8c2eb") - result = await consumer.single_chunk( - datasets=["612f6c39cbceda0ea9753d95"], - metrics=["org.dyamand.types.common.Temperature::number"], - from_timestamp=1740924034000, - to_timestamp=1741100614258, - limit=2 - ) - - assert len(result.items) == 2 diff --git a/src/obelisk/asynchronous/producer.py b/src/obelisk/asynchronous/producer.py deleted file mode 100644 index 697b217..0000000 --- a/src/obelisk/asynchronous/producer.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import List - -import httpx - -from obelisk.asynchronous.client import Client -from obelisk.exceptions import ObeliskError -from obelisk.types import IngestMode, TimestampPrecision - - -class Producer(Client): - """ - Allows publishing of data to Obelisk. - """ - - async def send(self, dataset: str, data: List[dict], - precision: TimestampPrecision = TimestampPrecision.MILLISECONDS, - mode: IngestMode = IngestMode.DEFAULT) -> httpx.Response: - """ - Publishes data to Obelisk - - Parameters - ---------- - dataset : str - ID for the dataset to publish to - data : List[dict] - List of Obelisk-acceptable datapoints. - Exact format varies between Classic or HFS, - caller is responsible for formatting. - precision : :class:`~obelisk.types.TimestampPrecision` = TimestampPrecision.MILLISECONDS - Precision used in the numeric timestamps contained in data. - Ensure it matches to avoid weird errors. - mode : :class:`~obelisk.types.IngestMode` = IngestMode.DEFAULT - See docs for :class:`~obelisk.types.IngestMode`. - - Raises - ------ - - ObeliskError - When the resulting status code is not 204, an empty :exc:`~obelisk.exceptions.ObeliskError` is raised. - """ - - params = { - 'datasetId': dataset, - 'timestampPrecision': precision.value, - 'mode': mode.value - } - - response = await self.http_post(f'{self._ingest_url}/{dataset}', data=data, - params=params) - if response.status_code != 204: - msg = f'An error occured during data ingest. Status {response.status_code}, message: {response.text}' - self.log.warning(msg) - raise ObeliskError(msg) - return response diff --git a/src/obelisk/sync/__init__.py b/src/obelisk/sync/__init__.py index 801ac08..66cf554 100644 --- a/src/obelisk/sync/__init__.py +++ b/src/obelisk/sync/__init__.py @@ -1,9 +1,7 @@ """ -This module provides wrappers for the classes in `obelisk.asynchronous` with a synchronous API. +This module provides wrappers for the classes in :mod:`obelisk.asynchronous` with a synchronous API. These hold on to a private event loop and block until a result is available. -There is no synchronous alternative to `obelisk.asynchronous.client.Client`. - Note ---- @@ -11,3 +9,5 @@ This is because it is internally nothing more than a wrapper over the asynchronous implementation. Use the asynchronous implementation in these situations. """ +__all__ = ["Obelisk"] +from .client import Obelisk diff --git a/src/obelisk/sync/consumer.py b/src/obelisk/sync/client.py similarity index 54% rename from src/obelisk/sync/consumer.py rename to src/obelisk/sync/client.py index 84e7cad..9aa4508 100644 --- a/src/obelisk/sync/consumer.py +++ b/src/obelisk/sync/client.py @@ -1,16 +1,22 @@ import asyncio from datetime import datetime, timedelta -from typing import List, Literal, Generator, Optional from math import floor +from typing import Generator, List, Literal, Optional -from obelisk.asynchronous.consumer import \ - Consumer as AsyncConsumer -from obelisk.strategies.retry import RetryStrategy, \ - NoRetryStrategy -from obelisk.types import QueryResult, Datapoint, ObeliskKind +import httpx +from obelisk.asynchronous import Obelisk as AsyncObelisk +from obelisk.strategies.retry import NoRetryStrategy, RetryStrategy +from obelisk.types import ( + Datapoint, + IngestMode, + ObeliskKind, + QueryResult, + TimestampPrecision, +) -class Consumer: + +class Obelisk: """ Component that contains all the logic to consume data from the Obelisk API (e.g. historical data, sse). @@ -23,22 +29,32 @@ class Consumer: loop: asyncio.AbstractEventLoop """Event loop used to run interal async operations""" - async_consumer: AsyncConsumer + async_obelisk: AsyncObelisk """The actual implementation this synchronous wrapper refers to""" - def __init__(self, client: str, secret: str, - retry_strategy: RetryStrategy = NoRetryStrategy(), - kind: ObeliskKind = ObeliskKind.CLASSIC): - self.async_consumer = AsyncConsumer(client, secret, retry_strategy, kind) + def __init__( + self, + client: str, + secret: str, + retry_strategy: RetryStrategy = NoRetryStrategy(), + kind: ObeliskKind = ObeliskKind.CLASSIC, + ): + self.async_obelisk = AsyncObelisk(client, secret, retry_strategy, kind) self.loop = asyncio.get_event_loop() - def single_chunk(self, datasets: List[str], metrics: Optional[List[str]] = None, - fields: Optional[dict] = None, - from_timestamp: Optional[int] = None, to_timestamp: Optional[int] = None, - order_by: Optional[dict] = None, - filter_: Optional[dict] = None, - limit: Optional[int] = None, limit_by: Optional[dict] = None, - cursor: Optional[str] = None) -> QueryResult: + def fetch_single_chunk( + self, + datasets: List[str], + metrics: Optional[List[str]] = None, + fields: Optional[dict] = None, + from_timestamp: Optional[int] = None, + to_timestamp: Optional[int] = None, + order_by: Optional[dict] = None, + filter_: Optional[dict] = None, + limit: Optional[int] = None, + limit_by: Optional[dict] = None, + cursor: Optional[str] = None, + ) -> QueryResult: """ Queries one chunk of events from Obelisk for given parameters, does not handle paging over Cursors. @@ -78,23 +94,36 @@ def single_chunk(self, datasets: List[str], metrics: Optional[List[str]] = None, used when paging through large result sets. """ - self.async_consumer.log.info("Starting task") + self.async_obelisk.log.info("Starting task") task = self.loop.create_task( - self.async_consumer.single_chunk(datasets, metrics, fields, from_timestamp, - to_timestamp, order_by, filter_, - limit, limit_by, cursor)) - self.async_consumer.log.info("Blocking...") + self.async_obelisk.fetch_single_chunk( + datasets, + metrics, + fields, + from_timestamp, + to_timestamp, + order_by, + filter_, + limit, + limit_by, + cursor, + ) + ) + self.async_obelisk.log.info("Blocking...") return self.loop.run_until_complete(task) - - def query(self, datasets: List[str], metrics: Optional[List[str]] = None, - fields: Optional[dict] = None, - from_timestamp: Optional[int] = None, - to_timestamp: Optional[int] = None, - order_by: Optional[dict] = None, - filter_: Optional[dict] = None, - limit: Optional[int] = None, - limit_by: Optional[dict] = None) -> List[Datapoint]: + def query( + self, + datasets: List[str], + metrics: Optional[List[str]] = None, + fields: Optional[dict] = None, + from_timestamp: Optional[int] = None, + to_timestamp: Optional[int] = None, + order_by: Optional[dict] = None, + filter_: Optional[dict] = None, + limit: Optional[int] = None, + limit_by: Optional[dict] = None, + ) -> List[Datapoint]: """ Queries data from obelisk, automatically iterating when a cursor is returned. @@ -131,19 +160,31 @@ def query(self, datasets: List[str], metrics: Optional[List[str]] = None, to a specified maximum number. """ - task = self.loop.create_task( - self.async_consumer.query(datasets, metrics, fields, from_timestamp, - to_timestamp, order_by, filter_, limit, - limit_by)) + self.async_obelisk.query( + datasets, + metrics, + fields, + from_timestamp, + to_timestamp, + order_by, + filter_, + limit, + limit_by, + ) + ) return self.loop.run_until_complete(task) - - def query_time_chunked(self, datasets: List[str], metrics: List[str], - from_time: datetime, to_time: datetime, - jump: timedelta, filter_: Optional[dict] = None, - direction: Literal['asc', 'desc'] = 'asc' - ) -> Generator[List[Datapoint], None, None]: + def query_time_chunked( + self, + datasets: List[str], + metrics: List[str], + from_time: datetime, + to_time: datetime, + jump: timedelta, + filter_: Optional[dict] = None, + direction: Literal["asc", "desc"] = "asc", + ) -> Generator[List[Datapoint], None, None]: """ Fetches all data matching the provided filters, yielding one chunk at a time. @@ -171,9 +212,48 @@ def query_time_chunked(self, datasets: List[str], metrics: List[str], current_start = from_time while current_start < to_time: - yield self.query(datasets=datasets, metrics=metrics, - from_timestamp=floor(current_start.timestamp() * 1000), - to_timestamp=floor((current_start + jump).timestamp() * 1000 - 1), - order_by={"field": ["timestamp"], "ordering": direction}, - filter_=filter_) + yield self.query( + datasets=datasets, + metrics=metrics, + from_timestamp=floor(current_start.timestamp() * 1000), + to_timestamp=floor((current_start + jump).timestamp() * 1000 - 1), + order_by={"field": ["timestamp"], "ordering": direction}, + filter_=filter_, + ) current_start += jump + + def send( + self, + dataset: str, + data: List[dict], + precision: TimestampPrecision = TimestampPrecision.MILLISECONDS, + mode: IngestMode = IngestMode.DEFAULT, + ) -> httpx.Response: + """ + Publishes data to Obelisk + + Parameters + ---------- + dataset : str + ID for the dataset to publish to + data : List[dict] + List of Obelisk-acceptable datapoints. + Exact format varies between Classic or HFS, + caller is responsible for formatting. + precision : TimestampPrecision = TimestampPrecision.MILLISECONDS + Precision used in the numeric timestamps contained in data. + Ensure it matches to avoid weird errors. + mode : IngestMode = IngestMode.DEFAULT + See docs for :class:`~obelisk.types.IngestMode`. + + Raises + ------ + + ObeliskError + When the resulting status code is not 204, an empty :exc:`~obelisk.exceptions.ObeliskError` is raised. + """ + + task = self.loop.create_task( + self.async_obelisk.send(dataset, data, precision, mode) + ) + return self.loop.run_until_complete(task) diff --git a/src/obelisk/sync/producer.py b/src/obelisk/sync/producer.py deleted file mode 100644 index 80ad66d..0000000 --- a/src/obelisk/sync/producer.py +++ /dev/null @@ -1,58 +0,0 @@ -import asyncio -from typing import List - -import httpx - -from obelisk.asynchronous.producer import \ - Producer as AsyncProducer -from obelisk.strategies.retry import RetryStrategy, \ - NoRetryStrategy -from obelisk.types import IngestMode, TimestampPrecision, \ - ObeliskKind - - -class Producer: - """ - Synchronous equivalient of :class:`~obelisk.asynchronous.producer.Producer`, - to publish data to Obelisk. - """ - - loop: asyncio.AbstractEventLoop - async_producer: AsyncProducer - - def __init__(self, client: str, secret: str, - retry_strategy: RetryStrategy = NoRetryStrategy(), - kind: ObeliskKind = ObeliskKind.CLASSIC): - self.async_producer = AsyncProducer(client, secret, retry_strategy, kind) - self.loop = asyncio.get_event_loop() - - def send(self, dataset: str, data: List[dict], - precision: TimestampPrecision = TimestampPrecision.MILLISECONDS, - mode: IngestMode = IngestMode.DEFAULT) -> httpx.Response: - """ - Publishes data to Obelisk - - Parameters - ---------- - dataset : str - ID for the dataset to publish to - data : List[dict] - List of Obelisk-acceptable datapoints. - Exact format varies between Classic or HFS, - caller is responsible for formatting. - precision : TimestampPrecision = TimestampPrecision.MILLISECONDS - Precision used in the numeric timestamps contained in data. - Ensure it matches to avoid weird errors. - mode : IngestMode = IngestMode.DEFAULT - See docs for :class:`~obelisk.types.IngestMode`. - - Raises - ------ - - ObeliskError - When the resulting status code is not 204, an empty :exc:`~obelisk.exceptions.ObeliskError` is raised. - """ - - task = self.loop.create_task( - self.async_producer.send(dataset, data, precision, mode)) - return self.loop.run_until_complete(task) diff --git a/src/obelisk/types.py b/src/obelisk/types.py index f4ce39a..af4faf0 100644 --- a/src/obelisk/types.py +++ b/src/obelisk/types.py @@ -53,3 +53,64 @@ class QueryResult(BaseModel): class ObeliskKind(str, Enum): CLASSIC = 'classic' HFS = 'hfs' + CORE = 'core' + + @property + def token_url(self) -> str: + match self: + case ObeliskKind.CLASSIC: + return 'https://obelisk.ilabt.imec.be/api/v3/auth/token' + case ObeliskKind.HFS: + return 'https://obelisk-hfs.discover.ilabt.imec.be/auth/realms/obelisk-hfs/protocol/openid-connect/token' + case ObeliskKind.CORE: + raise NotImplementedError() + + @property + def root_url(self) -> str: + match self: + case ObeliskKind.CLASSIC: + return 'https://obelisk.ilabt.imec.be/api/v3' + case ObeliskKind.HFS: + return 'https://obelisk-hfs.discover.ilabt.imec.be' + case ObeliskKind.CORE: + raise NotImplementedError() + + @property + def query_url(self) -> str: + match self: + case ObeliskKind.CLASSIC: + return 'https://obelisk.ilabt.imec.be/api/v3/data/query/events' + case ObeliskKind.HFS: + return 'https://obelisk-hfs.discover.ilabt.imec.be/data/query/events' + case ObeliskKind.CORE: + raise NotImplementedError() + + @property + def ingest_url(self) -> str: + match self: + case ObeliskKind.CLASSIC: + return 'https://obelisk.ilabt.imec.be/api/v3/data/ingest' + case ObeliskKind.HFS: + return 'https://obelisk-hfs.discover.ilabt.imec.be/data/ingest' + case ObeliskKind.CORE: + raise NotImplementedError() + + @property + def stream_url(self) -> str | None: + match self: + case ObeliskKind.CLASSIC: + return 'https://obelisk.ilabt.imec.be/api/v3/data/streams' + case ObeliskKind.HFS: + return None + case ObeliskKind.CORE: + raise NotImplementedError() + + @property + def use_json_auth(self) -> bool: + match self: + case ObeliskKind.CLASSIC: + return True + case ObeliskKind.HFS: + return False + case ObeliskKind.CORE: + return False diff --git a/src/tests/asynchronous/__init__.py b/src/tests/asynchronous/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tests/asynchronous/client_test.py b/src/tests/asynchronous/client_test.py new file mode 100644 index 0000000..f6ff4a7 --- /dev/null +++ b/src/tests/asynchronous/client_test.py @@ -0,0 +1,21 @@ +import pytest +from obelisk.asynchronous import Obelisk + +# Intentionally public client, restricted to public datasets. +client_id = "682c6c46604b3b3be35429df" +client_secret = "7136832d-01be-456a-a1fe-25c7f9e130c5" + +pytest_plugins = ('pytest_asyncio',) + +@pytest.mark.asyncio +async def test_fetch_demo_igent(): + consumer = Obelisk(client=client_id, secret=client_secret) + result = await consumer.fetch_single_chunk( + datasets=["612f6c39cbceda0ea9753d95"], + metrics=["org.dyamand.types.common.Temperature::number"], + from_timestamp=1740924034000, + to_timestamp=1741100614258, + limit=2 + ) + + assert len(result.items) == 2 diff --git a/src/tests/sync/__init__.py b/src/tests/sync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/obelisk/sync/consumer_test.py b/src/tests/sync/client_test.py similarity index 60% rename from src/obelisk/sync/consumer_test.py rename to src/tests/sync/client_test.py index ca6d308..cecc4ed 100644 --- a/src/obelisk/sync/consumer_test.py +++ b/src/tests/sync/client_test.py @@ -1,8 +1,11 @@ -from .consumer import Consumer +from obelisk.sync import Obelisk -def test_demo_igent(): - consumer = Consumer(client="67c716e616c11421cfe2faf6", secret="08dafe89-0389-45b4-9832-cc565fb8c2eb") - result = consumer.single_chunk( +client_id = "682c6c46604b3b3be35429df" +client_secret = "7136832d-01be-456a-a1fe-25c7f9e130c5" + +def test_demo_igent_fetch(): + consumer = Obelisk(client=client_id, secret=client_secret) + result = consumer.fetch_single_chunk( datasets=["612f6c39cbceda0ea9753d95"], metrics=["org.dyamand.types.common.Temperature::number"], from_timestamp=1740924034000, @@ -13,16 +16,16 @@ def test_demo_igent(): assert len(result.items) == 2 def test_two_instances(): - consumer_one = Consumer(client="67c716e616c11421cfe2faf6", secret="08dafe89-0389-45b4-9832-cc565fb8c2eb") - consumer_two = Consumer(client="67c716e616c11421cfe2faf6", secret="08dafe89-0389-45b4-9832-cc565fb8c2eb") - result_one = consumer_one.single_chunk( + consumer_one = Obelisk(client=client_id, secret=client_secret) + consumer_two = Obelisk(client=client_id, secret=client_secret) + result_one = consumer_one.fetch_single_chunk( datasets=["612f6c39cbceda0ea9753d95"], metrics=["org.dyamand.types.common.Temperature::number"], from_timestamp=1740924034000, to_timestamp=1741100614258, limit=2 ) - result_two = consumer_one.single_chunk( + result_two = consumer_two.fetch_single_chunk( datasets=["612f6c39cbceda0ea9753d95"], metrics=["org.dyamand.types.common.Temperature::number"], from_timestamp=1740924034000,