introduce ruff

Makefile

@@ -20,20 +20,23 @@ install-dev-deps:
 	pip3 install -r requirements/ci-requirements.txt
 
 pre-commit:
-	python3 -m flake8
-	python3 -m mypy sematic
-	python3 -m black sematic --check
+	uvx ruff format --check
+	uvx ruff check --fix sematic
+	uv run mypy -p sematic --disable-error-code import-untyped
 	pushd sematic/ui && npm run lint && popd
 
 fix:
-	black sematic
+	uvx ruff format
+	uvx ruff check --fix --show-fixes sematic
 
 .PHONY: py-prep
 py-prep:
 	uv --version || curl -LsSf https://astral.sh/uv/install.sh | sh
 	rm -rf ".venv" || echo "No virtualenv yet"
 	uv venv --python 3.12
 	uv sync --extra examples --extra ray
+	uv tool install --force ruff==0.8.3
+	uv pip install mypy==1.13.0
 
 .PHONY: py-sync
 py-sync:

bazel/worker.py

@@ -5,6 +5,7 @@
 //sematic/resolvers:worker we want to use the one in the wheel that the user has
 installed.
 """
+
 from sematic.resolvers.worker import wrap_main_with_logging
 
 if __name__ == "__main__":

pyproject.toml

@@ -149,6 +149,14 @@ lines-after-imports = 2
 [tool.pylsp.plugins.ruff]
 enabled = true
 
+[[tool.mypy.overrides]]
+module = "boto3.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "botocore.*"
+ignore_missing_imports = true
+
 [[tool.mypy.overrides]]
 module = "yaml.*"
 ignore_missing_imports = true
@@ -161,5 +169,8 @@ ignore_missing_imports = true
 module = "torch.*"
 ignore_missing_imports = true
 
+[tool.mypy]
+exclude = ".*site-packages.*"
+
 [tool.uv.sources]
 sematic = { workspace = true }

requirements/ci-requirements.txt

@@ -1,10 +1,7 @@
 
 # Linters and such. Pin them so that different devs
 # don't get different results from using them.
-flake8==7.1.1
 mypy==1.13.0
-black==24.10.0
-isort==5.13.2
 
 pytest
 # Required for pip-compile on the CI worker

sematic/__init__.py

@@ -7,6 +7,7 @@
 import platform  # noqa: E402
 import sys  # noqa: E402
 
+
 # `urllib` invokes the underlying OS framework to get configured system proxies.
 # On MacOS, this call causes the OS to immediately kill the `gunicorn` WSGI worker because
 # it had not immediately performed `exec()` after `fork()`.
@@ -63,6 +64,8 @@
 from sematic.runners.cloud_runner import CloudRunner  # noqa: F401,E402
 from sematic.runners.local_runner import LocalRunner, RerunMode  # noqa: F401,E402
 from sematic.runners.silent_runner import SilentRunner  # noqa: F401,E402
-from sematic.utils.exceptions import KubernetesError  # noqa: F401,E402
-from sematic.utils.exceptions import PipelineRunError  # noqa: F401,E402
+from sematic.utils.exceptions import (  # noqa: E402
+    KubernetesError,  # noqa: F401
+    PipelineRunError,  # noqa: F401
+)
 from sematic.versions import CURRENT_VERSION_STR as __version__  # noqa: F401,E402

sematic/abstract_plugin.py

@@ -18,6 +18,7 @@
 # Sematic
 from sematic.utils.exceptions import MissingPluginError
 
+
 logger = logging.getLogger(__name__)
 
 

sematic/abstract_system_metric.py

@@ -24,6 +24,7 @@
 from sematic.plugins.metrics_storage.sql.sql_metrics_storage import SQLMetricsStorage
 from sematic.utils.exceptions import DataIntegrityError
 
+
 MeasuredValue = Tuple[datetime, float]
 
 _BACKFILL_PAGE_SIZE = 100

sematic/api/app.py

@@ -2,6 +2,7 @@
 from flask import Flask
 from flask_cors import CORS
 
+
 sematic_api = Flask(__name__, static_folder="../ui/build/static")
 
 CORS(sematic_api, resources={r"/api/*": {"origins": "*"}})

sematic/api/endpoints/artifacts.py

@@ -19,13 +19,13 @@
 from sematic.db.models.user import User
 from sematic.db.queries import get_artifact
 
+
 logger = logging.getLogger(__name__)
 
 
 @sematic_api.route("/api/v1/artifacts", methods=["GET"])
 @authenticate
 def list_artifacts_endpoint(user: Optional[User] = None) -> flask.Response:
-
     parameters = get_request_parameters(args=flask.request.args, model=Artifact)
     limit, order, sql_predicates = (
         parameters.limit,

sematic/api/endpoints/auth.py

@@ -23,6 +23,7 @@
 from sematic.db.models.factories import make_user
 from sematic.db.queries import get_user_by_api_key, get_user_by_email, save_user
 
+
 # Email address for pseudo-user for the cron job that periodically
 # makes requests to the API to clean up dangling resources.
 CLEANER_EMAIL_ADDRESS = "cleaner@serviceaccount"
@@ -37,9 +38,7 @@ def authenticate_endpoint() -> flask.Response:
     friction, we let users run locally without authentication.
     """
     providers = {}
-    authenticate = get_bool_server_setting(
-        ServerSettingsVar.SEMATIC_AUTHENTICATE, False
-    )
+    authenticate = get_bool_server_setting(ServerSettingsVar.SEMATIC_AUTHENTICATE, False)
 
     if authenticate:
         for var in (

sematic/api/endpoints/events.py

@@ -16,6 +16,7 @@
 from sematic.api.endpoints.auth import authenticate, authenticate_starlette
 from sematic.db.models.user import User
 
+
 logger = logging.getLogger(__name__)
 
 _sio_server = None

sematic/api/endpoints/external_resources.py

@@ -25,6 +25,7 @@
     ResourceState,
 )
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -72,14 +73,9 @@ def get_resource_endpoint(user: Optional[User], resource_id: str) -> flask.Respo
     return flask.jsonify(payload)
 
 
-@sematic_api.route(
-    "/api/v1/external_resources/<resource_id>/activate", methods=["POST"]
-)
+@sematic_api.route("/api/v1/external_resources/<resource_id>/activate", methods=["POST"])
 @authenticate
-def activate_resource_endpoint(
-    user: Optional[User], resource_id: str
-) -> flask.Response:
-
+def activate_resource_endpoint(user: Optional[User], resource_id: str) -> flask.Response:
     record = get_external_resource_record(resource_id=resource_id)
     if record is None:
         return jsonify_error(
@@ -113,7 +109,6 @@ def activate_resource_endpoint(
 def deactivate_resource_endpoint(
     user: Optional[User], resource_id: str
 ) -> flask.Response:
-
     record = get_external_resource_record(resource_id=resource_id)
     if record is None:
         return jsonify_error(

sematic/api/endpoints/meta.py

@@ -17,6 +17,7 @@
 from sematic.db.models.user import User
 from sematic.versions import CURRENT_VERSION, MIN_CLIENT_SERVER_SUPPORTS
 
+
 logger = getLogger(__name__)
 
 

sematic/api/endpoints/metrics.py

@@ -30,6 +30,7 @@
 )
 from sematic.plugins.metrics_storage.sql.sql_metrics_storage import SQLMetricsStorage
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -59,9 +60,7 @@ def get_metric_endpoint(user: Optional[User], metric_name: str) -> flask.Respons
     try:
         labels: Dict = json.loads(flask.request.args.get("labels", "{}"))
     except Exception as e:
-        return jsonify_error(
-            f"Unable to deserialize labels: {e}", HTTPStatus.BAD_REQUEST
-        )
+        return jsonify_error(f"Unable to deserialize labels: {e}", HTTPStatus.BAD_REQUEST)
 
     try:
         rollup = _get_rollup(flask.request.args)
@@ -124,9 +123,7 @@ def list_metrics_endpoint(user: Optional[User]) -> flask.Response:
     try:
         labels: Dict = json.loads(flask.request.args.get("labels", "{}"))
     except Exception as e:
-        return jsonify_error(
-            f"Unable to deserialize labels: {e}", HTTPStatus.BAD_REQUEST
-        )
+        return jsonify_error(f"Unable to deserialize labels: {e}", HTTPStatus.BAD_REQUEST)
 
     metrics_storage = SQLMetricsStorage()
 
@@ -151,9 +148,7 @@ class MetricEvent(enum.IntEnum):
 }
 
 
-def save_event_metrics(
-    event: MetricEvent, runs: List[Run], user: Optional[User] = None
-):
+def save_event_metrics(event: MetricEvent, runs: List[Run], user: Optional[User] = None):
     """
     Compute and store System Metrics associated with a particular metric event.
     """

sematic/api/endpoints/payloads.py

@@ -11,6 +11,7 @@
 # Sematic
 from sematic.db.queries import get_user, get_users_by_id
 
+
 logger = logging.getLogger(__name__)
 
 

sematic/api/endpoints/request_parameters.py

@@ -26,6 +26,7 @@
 # Sematic
 from sematic.db.models.mixins.json_encodable_mixin import CONTAIN_FILTER_KEY
 
+
 logger = logging.getLogger(__name__)
 
 # Default page size
@@ -200,9 +201,7 @@ def list_garbage_ids(
         )
 
     scheme, netloc, path, _, fragment = urlsplit(request_url)
-    current_page_url = urlunsplit(
-        (scheme, netloc, path, encoded_request_args, fragment)
-    )
+    current_page_url = urlunsplit((scheme, netloc, path, encoded_request_args, fragment))
 
     ids = queries[garbage_filter]()
 
@@ -271,9 +270,7 @@ def _none_if_empty(name: str) -> Optional[str]:
         raise ValueError(f"Malformed filters: {filters_json}, error: {e}")
 
     sql_predicates = (
-        _get_sql_predicates(filters, column_mapping, model)
-        if len(filters) > 0
-        else None
+        _get_sql_predicates(filters, column_mapping, model) if len(filters) > 0 else None
     )
 
     order = ORDER_BY_DIRECTIONS.get(args.get("order", default_order))
@@ -438,9 +435,7 @@ def _extract_relationship_predicate(filter: ColumnPredicate, model: type):
         relationship_attribute.property,
         sqlalchemy.orm.relationships.RelationshipProperty,
     ):
-        raise ValueError(
-            f"{relationship_name} is not a relationship property of {model}"
-        )
+        raise ValueError(f"{relationship_name} is not a relationship property of {model}")
 
     relationship_model = relationship_attribute.property.mapper.class_
 

sematic/api/endpoints/resolutions.py

@@ -63,6 +63,7 @@
 from sematic.scheduling.kubernetes import cancel_job
 from sematic.utils.exceptions import ExceptionMetadata
 
+
 logger = logging.getLogger(__name__)
 
 _GARBAGE_COLLECTION_QUERIES: Dict[str, Callable[[], List[str]]] = {
@@ -296,9 +297,7 @@ def schedule_resolution_endpoint(
 
 @sematic_api.route("/api/v1/resolutions/<resolution_id>/rerun", methods=["POST"])
 @authenticate
-def rerun_resolution_endpoint(
-    user: Optional[User], resolution_id: str
-) -> flask.Response:
+def rerun_resolution_endpoint(user: Optional[User], resolution_id: str) -> flask.Response:
     original_resolution = get_resolution(resolution_id)
 
     if original_resolution.container_image_uri is None:
@@ -617,9 +616,7 @@ def clean_orphaned_resolution_jobs_endpoint(
 
 @sematic_api.route("/api/v1/resolutions/<root_id>/clean", methods=["POST"])
 @authenticate
-def clean_stale_resolution_endpoint(
-    user: Optional[User], root_id: str
-) -> flask.Response:
+def clean_stale_resolution_endpoint(user: Optional[User], root_id: str) -> flask.Response:
     resolution = get_resolution(root_id)
     root_run = get_run(root_id)
     logger.info(

sematic/api/endpoints/runs.py

@@ -60,6 +60,7 @@
 from sematic.utils.exceptions import ExceptionMetadata, IllegalStateTransitionError
 from sematic.utils.retry import retry
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -127,9 +128,7 @@ def list_runs_endpoint(user: Optional[User]) -> flask.Response:
     if len(garbage_filters) == 0:
         return _standard_list_runs(request_args)
 
-    logger.info(
-        "Searching for runs to garbage collect with filters: %s", garbage_filters
-    )
+    logger.info("Searching for runs to garbage collect with filters: %s", garbage_filters)
 
     if contained_extra_filters or len(garbage_filters) > 1:
         return jsonify_error(

sematic/api/endpoints/storage.py

@@ -16,6 +16,7 @@
 from sematic.plugins.abstract_storage import StorageDestination, get_storage_plugins
 from sematic.plugins.storage.local_storage import LocalStorage
 
+
 logger = logging.getLogger(__name__)