diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 45af396..56d2173 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Linting and unit tests +name: Lint, Build and Test on: push: branches: [ "main" ] @@ -15,10 +15,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python 3.12 + - name: Set up Python 3.9 uses: actions/setup-python@v3 with: - python-version: "3.12" + python-version: "3.9" - name: Install dependencies run: | @@ -32,8 +32,11 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide, we further relax this flake8 . --count --exit-zero --max-complexity=30 --max-line-length=130 --statistics - + - name: Build + run: | + pip install build + python -m build - name: Test with pytest run: | if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi - pytest --cov-report term --cov=src tests/ + pytest --cov-report term --cov=cs3client tests/ diff --git a/.gitignore b/.gitignore index 34de50d..4762aba 100644 --- a/.gitignore +++ b/.gitignore @@ -157,6 +157,9 @@ dmypy.json # Cython debug symbols cython_debug/ +# MacOS +.DS_Store + # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore diff --git a/README.md b/README.md index bdb0292..dfd58fb 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,7 @@ Alternatively, you can clone this repository and install manually: ```bash git clone git@github.com:cs3org/cs3-python-client.git cd cs3-python-client -pip install -e . -export PYTHONPATH="path/to/cs3-python-client/:$PYTHONPATH" +pip install . ``` @@ -89,12 +88,14 @@ ssl_verify = False # Optional, defaults to an empty string ssl_client_cert = test_client_cert # Optional, defaults to an empty string -ssl_client_key = test_client_key +ssl_client_key = test_client_key # Optional, defaults to an empty string -ssl_ca_cert = test_ca_cert +ssl_ca_cert = test_ca_cert # Optinal, defaults to an empty string auth_client_id = einstein +# Optional (can also be set when instansiating the class) +auth_client_secret = relativity # Optional, defaults to basic auth_login_type = basic @@ -112,73 +113,124 @@ lock_expiration = 1800 To use `cs3client`, you first need to import and configure it. Here's a simple example of how to set up and start using the client. For configuration see [Configuration](#configuration). For more in depth examples see `cs3-python-client/examples/`. -### Initilization +### Initilization and Authentication ```python import logging import configparser -from cs3client import CS3Client -from cs3resource import Resource +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) - log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) + ``` ### File Example ```python # mkdir -directory_resource = Resource.from_file_ref_and_endpoint(f"/eos/user/r/rwelande/test_directory") -res = client.file.make_dir(directory_resource) +directory_resource = Resource(abs_path=f"/eos/user/r/rwelande/test_directory") +res = client.file.make_dir(auth.get_token(), directory_resource) # touchfile -touch_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/touch_file.txt") -res = client.file.touch_file(touch_resource) +touch_resource = Resource(abs_path="/eos/user/r/rwelande/touch_file.txt") +res = client.file.touch_file(auth.get_token(), touch_resource) # setxattr -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") -res = client.file.set_xattr(resource, "iop.wopi.lastwritetime", str(1720696124)) +resource = Resource(abs_path="/eos/user/r/rwelande/text_file.txt") +res = client.file.set_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime", str(1720696124)) # rmxattr -res = client.file.remove_xattr(resource, "iop.wopi.lastwritetime") +res = client.file.remove_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime") # stat -res = client.file.stat(resource) +res = client.file.stat(auth.get_token(), resource) # removefile -res = client.file.remove_file(touch_resource) +res = client.file.remove_file(auth.get_token(), touch_resource) # rename -rename_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/rename_file.txt") -res = client.file.rename_file(resource, rename_resource) +rename_resource = Resource(abs_path="/eos/user/r/rwelande/rename_file.txt") +res = client.file.rename_file(auth.get_token(), resource, rename_resource) # writefile content = b"Hello World" size = len(content) -res = client.file.write_file(rename_resource, content, size) +res = client.file.write_file(auth.get_token(), rename_resource, content, size) # listdir -list_directory_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande") -res = client.file.list_dir(list_directory_resource) +list_directory_resource = Resource(abs_path="/eos/user/r/rwelande") +res = client.file.list_dir(auth.get_token(), list_directory_resource) # readfile -file_res = client.file.read_file(rename_resource) +file_res = client.file.read_file(auth.get_token(), rename_resource) +``` +### Lock Example +```python + +WEBDAV_LOCK_PREFIX = 'opaquelocktoken:797356a8-0500-4ceb-a8a0-c94c8cde7eba' + + +def encode_lock(lock): + '''Generates the lock payload for the storage given the raw metadata''' + if lock: + return WEBDAV_LOCK_PREFIX + ' ' + b64encode(lock.encode()).decode() + return None + +resource = Resource(abs_path="/eos/user/r/rwelande/lock_test.txt") + +# Set lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id=encode_lock("some_lock")) + +# Get lock +res = client.file.get_lock(auth_token, resource) +if res is not None: + lock_id = res["lock_id"] + print(res) + +# Unlock +res = client.file.unlock(auth_token, resource, app_name="a", lock_id=lock_id) + +# Refresh lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id=encode_lock("some_lock")) +res = client.file.refresh_lock( + auth_token, resource, app_name="a", lock_id=encode_lock("new_lock"), existing_lock_id=lock_id +) + +if res is not None: + print(res) + +res = client.file.get_lock(auth_token, resource) +if res is not None: + print(res) + ``` ### Share Example ```python # Create share # -resource = Resource.from_file_ref_and_endpoint("/eos/user/r//text.txt") -resource_info = client.file.stat(resource) +resource = Resource(abs_path="/eos/user/r//text.txt") +resource_info = client.file.stat(auth.get_token(), resource) user = client.user.get_user_by_claim("username", "") -res = client.share.create_share(resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER") +res = client.share.create_share(auth.get_token(), resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER") # List existing shares # filter_list = [] @@ -186,32 +238,32 @@ filter = client.share.create_share_filter(resource_id=resource_info.id, filter_t filter_list.append(filter) filter = client.share.create_share_filter(share_state="SHARE_STATE_PENDING", filter_type="TYPE_STATE") filter_list.append(filter) -res, _ = client.share.list_existing_shares() +res, _ = client.share.list_existing_shares(auth.get_token(), ) # Get share # share_id = "58" -res = client.share.get_share(opaque_id=share_id) +res = client.share.get_share(auth.get_token(), opaque_id=share_id) # update share # -res = client.share.update_share(opaque_id=share_id, role="VIEWER") +res = client.share.update_share(auth.get_token(), opaque_id=share_id, role="VIEWER") # remove share # -res = client.share.remove_share(opaque_id=share_id) +res = client.share.remove_share(auth.get_token(), opaque_id=share_id) # List existing received shares # filter_list = [] filter = client.share.create_share_filter(share_state="SHARE_STATE_ACCEPTED", filter_type="TYPE_STATE") filter_list.append(filter) -res, _ = client.share.list_received_existing_shares() +res, _ = client.share.list_received_existing_shares(auth.get_token()) # get received share # -received_share = client.share.get_received_share(opaque_id=share_id) +received_share = client.share.get_received_share(auth.get_token(), opaque_id=share_id) # update recieved share # -res = client.share.update_received_share(received_share=received_share, state="SHARE_STATE_ACCEPTED") +res = client.share.update_received_share(auth.get_token(), received_share=received_share, state="SHARE_STATE_ACCEPTED") # create public share # -res = client.share.create_public_share(resource_info, role="VIEWER") +res = client.share.create_public_share(auth.get_token(), resource_info, role="VIEWER") # list existing public shares # filter_list = [] @@ -219,22 +271,22 @@ filter = client.share.create_public_share_filter(resource_id=resource_info.id, f filter_list.append(filter) res, _ = client.share.list_existing_public_shares(filter_list=filter_list) -res = client.share.get_public_share(opaque_id=share_id, sign=True) +res = client.share.get_public_share(auth.get_token(), opaque_id=share_id, sign=True) # OR token = "" # res = client.share.get_public_share(token=token, sign=True) # update public share # -res = client.share.update_public_share(type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello") +res = client.share.update_public_share(auth.get_token(), type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello") # remove public share # -res = client.share.remove_public_share(token=token) +res = client.share.remove_public_share(auth.get_token(), token=token) ``` ### User Example ```python # find_user -res = client.user.find_users("rwel") +res = client.user.find_users(auth.get_token(), "rwel") # get_user res = client.user.get_user("https://auth.cern.ch/auth/realms/cern", "asdoiqwe") @@ -253,21 +305,21 @@ res = client.user.get_user_by_claim("username", "rwelande") ### App Example ```python # list_app_providers -res = client.app.list_app_providers() +res = client.app.list_app_providers(auth.get_token()) # open_in_app -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/collabora.odt") -res = client.app.open_in_app(resource) +resource = Resource(abs_path="/eos/user/r/rwelande/collabora.odt") +res = client.app.open_in_app(auth.get_token(), resource) ``` ### Checkpoint Example ```python # list file versions -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/test.md") -res = client.checkpoint.list_file_versions(resource) +resource = Resource(abs_path="/eos/user/r/rwelande/test.md") +res = client.checkpoint.list_file_versions(auth.get_token(), resource) # restore file version -res = client.checkpoint.restore_file_version(resource, "1722936250.0569fa2f") +res = client.checkpoint.restore_file_version(auth.get_token(), resource, "1722936250.0569fa2f") ``` ## Documentation diff --git a/src/__init__.py b/cs3client/__init__.py similarity index 100% rename from src/__init__.py rename to cs3client/__init__.py diff --git a/src/app.py b/cs3client/app.py similarity index 81% rename from src/app.py rename to cs3client/app.py index d54ce22..ea29c07 100644 --- a/src/app.py +++ b/cs3client/app.py @@ -3,19 +3,19 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging -from auth import Auth -from cs3resource import Resource import cs3.app.registry.v1beta1.registry_api_pb2 as cs3arreg import cs3.app.registry.v1beta1.resources_pb2 as cs3arres import cs3.gateway.v1beta1.gateway_api_pb2 as cs3gw import cs3.app.provider.v1beta1.resources_pb2 as cs3apr from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub -from statuscodehandler import StatusCodeHandler -from config import Config + +from .cs3resource import Resource +from .statuscodehandler import StatusCodeHandler +from .config import Config class App: @@ -28,7 +28,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -37,20 +36,21 @@ def __init__( :param config: Config object containing the configuration parameters. :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. :param status_code_handler: An instance of the StatusCodeHandler class. """ self._status_code_handler: StatusCodeHandler = status_code_handler self._gateway: GatewayAPIStub = gateway self._log: logging.Logger = log self._config: Config = config - self._auth: Auth = auth - def open_in_app(self, resource: Resource, view_mode: str = None, app: str = None) -> cs3apr.OpenInAppURL: + def open_in_app( + self, auth_token: tuple, resource: Resource, view_mode: str = None, app: str = None + ) -> cs3apr.OpenInAppURL: """ Open a file in an app, given the resource, view mode (VIEW_MODE_VIEW_ONLY, VIEW_MODE_READ_ONLY, VIEW_MODE_READ_WRITE, VIEW_MODE_PREVIEW), and app name. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource object containing the resource information. :param view_mode: View mode of the app. :param app: App name. @@ -63,21 +63,22 @@ def open_in_app(self, resource: Resource, view_mode: str = None, app: str = None if view_mode: view_mode_type = cs3gw.OpenInAppRequest.ViewMode.Value(view_mode) req = cs3gw.OpenInAppRequest(ref=resource.ref, view_mode=view_mode_type, app=app) - res = self._gateway.OpenInApp(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.OpenInApp(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "open in app", f"{resource.get_file_ref_str()}") self._log.debug(f'msg="Invoked OpenInApp" {resource.get_file_ref_str()} trace="{res.status.trace}"') return res.OpenInAppURL - def list_app_providers(self) -> list[cs3arres.ProviderInfo]: + def list_app_providers(self, auth_token: dict) -> list[cs3arres.ProviderInfo]: """ list_app_providers lists all the app providers. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :return: List of app providers. :raises: AuthenticationException (Operation not permitted) :raises: UnknownException (Unknown error) """ req = cs3arreg.ListAppProvidersRequest() - res = self._gateway.ListAppProviders(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListAppProviders(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list app providers") self._log.debug(f'msg="Invoked ListAppProviders" res_count="{len(res.providers)}" trace="{res.status.trace}"') return res.providers diff --git a/src/auth.py b/cs3client/auth.py similarity index 58% rename from src/auth.py rename to cs3client/auth.py index 94b0714..a594094 100644 --- a/src/auth.py +++ b/cs3client/auth.py @@ -3,20 +3,22 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import grpc import jwt import datetime import logging +from typing import Union from cs3.gateway.v1beta1.gateway_api_pb2 import AuthenticateRequest from cs3.auth.registry.v1beta1.registry_api_pb2 import ListAuthProvidersRequest from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub from cs3.rpc.v1beta1.code_pb2 import CODE_OK -from exceptions.exceptions import AuthenticationException, SecretNotSetException -from config import Config +from .cs3client import CS3Client +from .exceptions.exceptions import AuthenticationException, SecretNotSetException +from .config import Config class Auth: @@ -24,7 +26,7 @@ class Auth: Auth class to handle authentication and token validation with CS3 Gateway API. """ - def __init__(self, config: Config, log: logging.Logger, gateway: GatewayAPIStub) -> None: + def __init__(self, cs3_client: CS3Client) -> None: """ Initializes the Auth class with configuration, logger, and gateway stub, NOTE that token OR the client secret has to be set when instantiating the auth object. @@ -33,33 +35,34 @@ def __init__(self, config: Config, log: logging.Logger, gateway: GatewayAPIStub) :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. """ - self._gateway: GatewayAPIStub = gateway - self._log: logging.Logger = log - self._config: Config = config - # The user should be able to change the client secret (e.g. token) at runtime - self._client_secret: str | None = None - self._token: str | None = None - - def set_token(self, token: str) -> None: - """ - Should be used if the user wishes to set the reva token directly, instead of letting the client - exchange credentials for the token. NOTE that token OR the client secret has to be set when - instantiating the client object. - - :param token: The reva token. - """ - self._token = token + self._gateway: GatewayAPIStub = cs3_client._gateway + self._log: logging.Logger = cs3_client._log + self._config: Config = cs3_client._config + # The user should be able to change the client secret (e.g. token) and client id at runtime + self._client_secret: Union[str, None] = self._config.auth_client_secret + self._client_id: Union[str, None] = self._config.auth_client_id + self._token: Union[str, None] = None def set_client_secret(self, token: str) -> None: """ Sets the client secret, exists so that the user can change the client secret (e.g. token, password) at runtime, - without having to create a new Auth object. NOTE that token OR the client secret has to be set when - instantiating the client object. + without having to create a new Auth object. Note client secret has to be set when + instantiating the client object or through the configuration. :param token: Auth token/password. """ self._client_secret = token + def set_client_id(self, id: str) -> None: + """ + Sets the client id, exists so that the user can change the client id at runtime, without having to create + a new Auth object. Settings this (either through config or here) is optional unless you are using + basic authentication. + + :param token: id. + """ + self._client_id = id + def get_token(self) -> tuple[str, str]: """ Attempts to get a valid authentication token. If the token is not valid, a new token is requested @@ -71,31 +74,32 @@ def get_token(self) -> tuple[str, str]: :raises: SecretNotSetException (neither token or client secret was set) """ - if not Auth._check_token(self._token): - # Check that client secret or token is set - if not self._client_secret and not self._token: - self._log.error("Attempted to authenticate, neither client secret or token was set.") - raise SecretNotSetException("The client secret (e.g. token, passowrd) is not set") - elif not self._client_secret and self._token: - # Case where ONLY a token is provided but it has expired - self._log.error("The provided token have expired") - raise AuthenticationException("The credentials have expired") - # Create an authentication request + if not self._client_secret: + self._log.error("Attempted to authenticate, client secret was not set") + raise SecretNotSetException("The client secret (e.g. token, passowrd) is not set") + + try: + self.check_token(self._token) + except AuthenticationException: + # Token has expired or has not been set, obtain another one. req = AuthenticateRequest( type=self._config.auth_login_type, - client_id=self._config.auth_client_id, + client_id=self._client_id, client_secret=self._client_secret, ) # Send the authentication request to the CS3 Gateway res = self._gateway.Authenticate(req) if res.status.code != CODE_OK: - self._log.error(f"Failed to authenticate user {self._config.auth_client_id}, error: {res.status}") + self._log.error(f'msg="Failed to authenticate" ' + f'user="{self._client_id if self._client_id else "no_id_set"}" ' + f'error_code="{res.status}"') raise AuthenticationException( - f"Failed to authenticate user {self._config.auth_client_id}, error: {res.status}" + f'Failed to authenticate: user="{self._client_id if self._client_id else "no_id_set"}" ' + f'error_code="{res.status}"' ) self._token = res.token - self._log.debug(f'msg="Authenticated user" user="{self._config.auth_client_id}"') + self._log.debug(f'msg="Authenticated user" user="{self._client_id if self._client_id else "no_id_set"}"') return ("x-access-token", self._token) def list_auth_providers(self) -> list[str]: @@ -108,7 +112,7 @@ def list_auth_providers(self) -> list[str]: try: res = self._gateway.ListAuthProviders(request=ListAuthProvidersRequest()) if res.status.code != CODE_OK: - self._log.error(f"List auth providers request failed, error: {res.status}") + self._log.error(f'msg="List auth providers request failed" error_code="{res.status}"') raise Exception(res.status.message) except grpc.RpcError as e: self._log.error("List auth providers request failed") @@ -116,20 +120,22 @@ def list_auth_providers(self) -> list[str]: return res.types @classmethod - def _check_token(cls, token: str) -> bool: + def check_token(cls, token: str) -> tuple: """ Checks if the given token is set and valid. :param token: JWT token as a string. - :return: True if the token is valid, False otherwise. + :return tuple: A tuple containing the header key and the token. + :raises: ValueError (Token missing) + :raises: AuthenticationException (Token is expired) """ if not token: - return False + raise AuthenticationException("Token not set") # Decode the token without verifying the signature decoded_token = jwt.decode(jwt=token, algorithms=["HS256"], options={"verify_signature": False}) now = datetime.datetime.now().timestamp() token_expiration = decoded_token.get("exp") if token_expiration and now > token_expiration: - return False + raise AuthenticationException("Token has expired") - return True + return ("x-access-token", token) diff --git a/src/checkpoint.py b/cs3client/checkpoint.py similarity index 83% rename from src/checkpoint.py rename to cs3client/checkpoint.py index 9e4cbdd..dfedb00 100644 --- a/src/checkpoint.py +++ b/cs3client/checkpoint.py @@ -3,18 +3,18 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ from typing import Generator import logging -from auth import Auth import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3spp from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub -from config import Config -from statuscodehandler import StatusCodeHandler -from cs3resource import Resource + +from .config import Config +from .statuscodehandler import StatusCodeHandler +from .cs3resource import Resource class Checkpoint: @@ -27,7 +27,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -36,21 +35,20 @@ def __init__( :param config: Config object containing the configuration parameters. :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. :param status_code_handler: An instance of the StatusCodeHandler class. """ self._gateway: GatewayAPIStub = gateway self._log: logging.Logger = log self._config: Config = config - self._auth: Auth = auth self._status_code_handler: StatusCodeHandler = status_code_handler def list_file_versions( - self, resource: Resource, page_token: str = "", page_size: int = 0 + self, auth_token: tuple, resource: Resource, page_token: str = "", page_size: int = 0 ) -> Generator[cs3spr.FileVersion, any, any]: """ List all versions of a file. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource object containing the resource information. :param page_token: Token for pagination. :param page_size: Number of file versions to return. @@ -61,15 +59,18 @@ def list_file_versions( :raises: UnknownException (Unknown error) """ req = cs3spp.ListFileVersionsRequest(ref=resource.ref, page_token=page_token, page_size=page_size) - res = self._gateway.ListFileVersions(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListFileVersions(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list file versions", f"{resource.get_file_ref_str()}") self._log.debug(f'msg="list file versions" {resource.get_file_ref_str()} trace="{res.status.trace}"') return res.versions - def restore_file_version(self, resource: Resource, version_key: str, lock_id: str = None) -> None: + def restore_file_version( + self, auth_token: tuple, resource: Resource, version_key: str, lock_id: str = None + ) -> None: """ Restore a file to a previous version. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource object containing the resource information. :param version_key: Key of the version to restore. :param lock_id: Lock ID of the file (OPTIONAL). @@ -79,7 +80,7 @@ def restore_file_version(self, resource: Resource, version_key: str, lock_id: st :raises: UnknownException (Unknown error) """ req = cs3spp.RestoreFileVersionRequest(ref=resource.ref, key=version_key, lock_id=lock_id) - res = self._gateway.RestoreFileVersion(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.RestoreFileVersion(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "restore file version", f"{resource.get_file_ref_str()}") self._log.debug(f'msg="restore file version" {resource.get_file_ref_str()} trace="{res.status.trace}"') return diff --git a/src/config.py b/cs3client/config.py similarity index 95% rename from src/config.py rename to cs3client/config.py index 316b204..78ba99b 100644 --- a/src/config.py +++ b/cs3client/config.py @@ -131,6 +131,15 @@ def auth_client_id(self) -> str: """ return self._config.get(self._config_category, "auth_client_id", fallback=None) + @property + def auth_client_secret(self) -> str: + """ + The auth_client_secret property returns the auth_client_secret value from the configuration, + + :return: auth_client_secret + """ + return self._config.get(self._config_category, "auth_client_secret", fallback=None) + @property def tus_enabled(self) -> bool: """ diff --git a/src/cs3client.py b/cs3client/cs3client.py similarity index 80% rename from src/cs3client.py rename to cs3client/cs3client.py index de2e523..79cb608 100644 --- a/src/cs3client.py +++ b/cs3client/cs3client.py @@ -3,22 +3,21 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import grpc import logging import cs3.gateway.v1beta1.gateway_api_pb2_grpc as cs3gw_grpc - from configparser import ConfigParser -from auth import Auth -from file import File -from user import User -from share import Share -from statuscodehandler import StatusCodeHandler -from app import App -from checkpoint import Checkpoint -from config import Config + +from .file import File +from .user import User +from .share import Share +from .statuscodehandler import StatusCodeHandler +from .app import App +from .checkpoint import Checkpoint +from .config import Config class CS3Client: @@ -46,14 +45,13 @@ def __init__(self, config: ConfigParser, config_category: str, log: logging.Logg self._gateway: cs3gw_grpc.GatewayAPIStub = cs3gw_grpc.GatewayAPIStub(self.channel) self._status_code_handler: StatusCodeHandler = StatusCodeHandler(self._log, self._config) - self.auth: Auth = Auth(self._config, self._log, self._gateway) - self.file: File = File(self._config, self._log, self._gateway, self.auth, self._status_code_handler) - self.user: User = User(self._config, self._log, self._gateway, self.auth, self._status_code_handler) - self.app: App = App(self._config, self._log, self._gateway, self.auth, self._status_code_handler) + self.file: File = File(self._config, self._log, self._gateway, self._status_code_handler) + self.user: User = User(self._config, self._log, self._gateway, self._status_code_handler) + self.app: App = App(self._config, self._log, self._gateway, self._status_code_handler) self.checkpoint: Checkpoint = Checkpoint( - self._config, self._log, self._gateway, self.auth, self._status_code_handler + self._config, self._log, self._gateway, self._status_code_handler ) - self.share = Share(self._config, self._log, self._gateway, self.auth, self._status_code_handler) + self.share = Share(self._config, self._log, self._gateway, self._status_code_handler) def _create_channel(self) -> grpc.Channel: """ diff --git a/src/cs3resource.py b/cs3client/cs3resource.py similarity index 90% rename from src/cs3resource.py rename to cs3client/cs3resource.py index 6a19ee3..91097b5 100644 --- a/src/cs3resource.py +++ b/cs3client/cs3resource.py @@ -7,6 +7,7 @@ """ import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr +from typing import Union class Resource: @@ -26,12 +27,12 @@ class Resource: def __init__( self, - abs_path: str | None = None, - rel_path: str | None = None, - opaque_id: str | None = None, - parent_id: str | None = None, - storage_id: str | None = None, - space_id: str | None = None, + abs_path: Union[str, None] = None, + rel_path: Union[str, None] = None, + opaque_id: Union[str, None] = None, + parent_id: Union[str, None] = None, + storage_id: Union[str, None] = None, + space_id: Union[str, None] = None, ) -> None: """ initializes the Resource class, either abs_path, rel_path or opaque_id is required @@ -45,15 +46,15 @@ def __init__( :param storage_id: storage id (optional) :param space_id: space id (optional) """ - self._abs_path: str | None = abs_path - self._rel_path: str | None = rel_path - self._parent_id: str | None = parent_id - self._opaque_id: str | None = opaque_id - self._space_id: str | None = space_id - self._storage_id: str | None = storage_id + self._abs_path: Union[str, None] = abs_path + self._rel_path: Union[str, None] = rel_path + self._parent_id: Union[str, None] = parent_id + self._opaque_id: Union[str, None] = opaque_id + self._space_id: Union[str, None] = space_id + self._storage_id: Union[str, None] = storage_id @classmethod - def from_file_ref_and_endpoint(cls, file: str, endpoint: str | None = None) -> "Resource": + def from_file_ref_and_endpoint(cls, file: str, endpoint: Union[str, None] = None) -> "Resource": """ Extracts the attributes from the file and endpoint and returns a resource. diff --git a/src/exceptions/__init__.py b/cs3client/exceptions/__init__.py similarity index 100% rename from src/exceptions/__init__.py rename to cs3client/exceptions/__init__.py diff --git a/src/exceptions/exceptions.py b/cs3client/exceptions/exceptions.py similarity index 100% rename from src/exceptions/exceptions.py rename to cs3client/exceptions/exceptions.py diff --git a/cs3client/file.py b/cs3client/file.py new file mode 100644 index 0000000..ee08b82 --- /dev/null +++ b/cs3client/file.py @@ -0,0 +1,628 @@ +""" +file.py + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 30/08/2024 +""" + +import time +import logging +import http +import requests +from typing import Union +from typing import Generator +import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr +import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3sp +from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub +import cs3.types.v1beta1.types_pb2 as types +import cs3.rpc.v1beta1.code_pb2 as cs3code + + +from .config import Config +from .exceptions.exceptions import AuthenticationException, FileLockedException +from .cs3resource import Resource +from .statuscodehandler import StatusCodeHandler + +LOCK_ATTR_KEY = 'cs3client.advlock' + + +class File: + """ + File class to interact with the CS3 API. + """ + + def __init__( + self, config: Config, log: logging.Logger, gateway: GatewayAPIStub, + status_code_handler: StatusCodeHandler + ) -> None: + """ + Initializes the File class with configuration, logger, auth, gateway stub, and status code handler. + + :param config: Config object containing the configuration parameters + :param log: Logger instance for logging + :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway + :param status_code_handler: An instance of the StatusCodeHandler class + """ + self._config: Config = config + self._log: logging.Logger = log + self._gateway: GatewayAPIStub = gateway + self._status_code_handler: StatusCodeHandler = status_code_handler + + def stat(self, auth_token: tuple, resource: Resource) -> cs3spr.ResourceInfo: + """ + Stat a file and return the ResourceInfo object. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: resource to stat + :return: cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo (success) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + + """ + tstart = time.time() + res = self._gateway.Stat(request=cs3sp.StatRequest(ref=resource.ref), metadata=[auth_token]) + tend = time.time() + self._status_code_handler.handle_errors(res.status, "stat", resource.get_file_ref_str()) + self._log.info( + f'msg="Invoked Stat" fileref="{resource.ref}" {resource.get_file_ref_str()} trace="{res.status.trace}" ' + f'elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + return res.info + + def set_xattr(self, auth_token: tuple, resource: Resource, key: str, value: str, lock_id: str = None) -> None: + """ + Set the extended attribute to for a resource. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: resource that has the attribute. + :param key: attribute key + :param value: value to set + :param lock_id: lock id + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown error) + """ + md = cs3spr.ArbitraryMetadata() + md.metadata.update({key: value}) # pylint: disable=no-member + req = cs3sp.SetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata=md, lock_id=lock_id) + res = self._gateway.SetArbitraryMetadata(request=req, metadata=[auth_token]) + # CS3 storages may refuse to set an xattr in case of lock mismatch: this is an overprotection, + # as the lock should concern the file's content, not its metadata, however we need to handle that + self._status_code_handler.handle_errors(res.status, "set extended attribute", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked setxattr" trace="{res.status.trace}"') + + def remove_xattr(self, auth_token: tuple, resource: Resource, key: str, lock_id: str = None) -> None: + """ + Remove the extended attribute . + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: cs3client resource + :param key: key for attribute to remove + :param lock_id: lock id + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: AuthenticationException (Authentication failed) + :raises: UnknownException (Unknown error) + """ + req = cs3sp.UnsetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata_keys=[key], lock_id=lock_id) + res = self._gateway.UnsetArbitraryMetadata(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "remove extended attribute", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked UnsetArbitraryMetaData" trace="{res.status.trace}"') + + def rename_file( + self, auth_token: tuple, resource: Resource, newresource: Resource, lock_id: str = None + ) -> None: + """ + Rename/move resource to new resource. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Original resource + :param newresource: New resource + :param lock_id: lock id + :return: None (Success) + :raises: NotFoundException (Original resource not found) + :raises: FileLockException (Resource is locked) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + req = cs3sp.MoveRequest(source=resource.ref, destination=newresource.ref, lock_id=lock_id) + res = self._gateway.Move(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "rename file", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked Move" trace="{res.status.trace}"') + + def remove_file(self, auth_token: tuple, resource: Resource, lock_id: str = None) -> None: + """ + Remove a resource. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to remove + :param lock_id: lock id + :return: None (Success) + :raises: AuthenticationException (Authentication Failed) + :raises: NotFoundException (Resource not found) + :raises: UnknownException (Unknown error) + """ + req = cs3sp.DeleteRequest(ref=resource.ref, lock_id=lock_id) + res = self._gateway.Delete(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "remove file", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked Delete" trace="{res.status.trace}"') + + def touch_file(self, auth_token: tuple, resource: Resource) -> None: + """ + Create a resource. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to create + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown error) + """ + req = cs3sp.TouchFileRequest( + ref=resource.ref, + opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode("0"))}), + ) + res = self._gateway.TouchFile(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "touch file", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked TouchFile" trace="{res.status.trace}"') + + def write_file( + self, auth_token: tuple, resource: Resource, content: Union[str, bytes], size: int, + lock_md: tuple = ('', '') + ) -> None: + """ + Write a file using the given userid as access token. The entire content is written + and any pre-existing file is deleted (or moved to the previous version if supported), + writing a file with size 0 is equivalent to "touch file" and should be used if the + implementation does not support touchfile. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to write content to + :param content: content to write + :param size: size of content (optional) + :param lock_md: tuple (, ) + :return: None (Success) + :raises: FileLockedException (File is locked), + :raises: AuthenticationException (Authentication failed) + :raises: UnknownException (Unknown error) + + """ + app_name, lock_id = lock_md + tstart = time.time() + # prepare endpoint + if size == -1: + if isinstance(content, str): + content = bytes(content, "UTF-8") + size = len(content) + req = cs3sp.InitiateFileUploadRequest( + ref=resource.ref, + opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode(str(size)))}), + lock_id=lock_id + ) + res = self._gateway.InitiateFileUpload(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "write file", resource.get_file_ref_str()) + tend = time.time() + self._log.debug( + f'msg="Invoked InitiateFileUpload" trace="{res.status.trace}" protocols="{res.protocols}"' + ) + + # Upload + try: + protocol = [p for p in res.protocols if p.protocol in ["simple", "spaces"]][0] + if self._config.tus_enabled: + headers = { + "Tus-Resumable": "1.0.0", + "File-Path": resource.file, + "File-Size": str(size), + "X-Reva-Transfer": protocol.token, + **dict([auth_token]), + "X-Lock-Id": lock_id, + "X-Lock_Holder": app_name, + } + else: + headers = { + "Upload-Length": str(size), + "X-Reva-Transfer": protocol.token, + **dict([auth_token]), + "X-Lock-Id": lock_id, + "X-Lock_Holder": app_name, + } + putres = requests.put( + url=protocol.upload_endpoint, + data=content, + headers=headers, + verify=self._config.ssl_verify, + timeout=self._config.http_timeout, + ) + except requests.exceptions.RequestException as e: + self._log.error(f'msg="Exception when uploading file to Reva" reason="{e}"') + raise IOError(e) from e + if putres.status_code == http.client.CONFLICT: + self._log.info( + f'msg="Got conflict on PUT, file is locked" reason="{putres.reason}" {resource.get_file_ref_str()}' + ) + raise FileLockedException(f"Lock mismatch or lock expired: {putres.reason}") + if putres.status_code == http.client.UNAUTHORIZED: + self._log.warning( + f'msg="Authentication failed on write" reason="{putres.reason}" {resource.get_file_ref_str()}' + ) + raise AuthenticationException(f"Operation not permitted: {putres.reason}") + if putres.status_code != http.client.OK: + if ( + size == 0 + ): # 0-byte file uploads may have been finalized after InitiateFileUploadRequest, let's assume it's OK + # Should use TouchFileRequest instead + self._log.info( + f'msg="0-byte file written successfully" {resource.get_file_ref_str()} ' + f' elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + return + + self._log.error( + f'msg="Error uploading file" code="{putres.status_code}" reason="{putres.reason}"' + ) + raise IOError(putres.reason) + self._log.info( + f'msg="File written successfully" {resource.get_file_ref_str()} ' + f'elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + + def read_file(self, auth_token: tuple, resource: Resource, lock_id: str = None) -> Generator[bytes, None, None]: + """ + Read a file. Note that the function is a generator, managed by the app server. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to read + :param lock_id: lock id + :return: Generator[Bytes, None, None] (Success) + :raises: NotFoundException (Resource not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + tstart = time.time() + + # prepare endpoint + req = cs3sp.InitiateFileDownloadRequest(ref=resource.ref, lock_id=lock_id) + res = self._gateway.InitiateFileDownload(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "read file", resource.get_file_ref_str()) + tend = time.time() + self._log.debug( + f'msg="Invoked InitiateFileDownload" trace="{res.status.trace}" protocols="{res.protocols}"' + ) + + # Download + try: + protocol = [p for p in res.protocols if p.protocol in ["simple", "spaces"]][0] + headers = {"X-Reva-Transfer": protocol.token, **dict([auth_token])} + fileget = requests.get( + url=protocol.download_endpoint, + headers=headers, + verify=self._config.ssl_verify, + timeout=self._config.http_timeout, + stream=True, + ) + except requests.exceptions.RequestException as e: + self._log.error(f'msg="Exception when downloading file from Reva" reason="{e}"') + raise IOError(e) + data = fileget.iter_content(self._config.chunk_size) + if fileget.status_code != http.client.OK: + # status.message.replace('"', "'") is not allowed inside f strings python<3.12 + status_msg = fileget.reason.replace('"', "'") + self._log.error( + f'msg="Error downloading file from Reva" code="{fileget.status_code}" ' + f'reason="{status_msg}"' + ) + raise IOError(fileget.reason) + else: + self._log.info( + f'msg="File open for read" {resource.get_file_ref_str()} elapsedTimems="{(tend - tstart) * 1000:.1f}"' + ) + for chunk in data: + yield chunk + + def make_dir(self, auth_token: tuple, resource: Resource) -> None: + """ + Create a directory. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Direcotry to create + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: AuthenticationException (Authentication failed) + :raises: UnknownException (Unknown error) + """ + req = cs3sp.CreateContainerRequest(ref=resource.ref) + res = self._gateway.CreateContainer(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "make directory", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked CreateContainer" trace="{res.status.trace}"') + + def list_dir( + self, auth_token: tuple, resource: Resource + ) -> Generator[cs3spr.ResourceInfo, None, None]: + """ + List the contents of a directory, note that the function is a generator. + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: the directory + :return: Generator[cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo, None, None] (Success) + :raises: NotFoundException (Resrouce not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown error) + """ + req = cs3sp.ListContainerRequest(ref=resource.ref) + res = self._gateway.ListContainer(request=req, metadata=[auth_token]) + self._status_code_handler.handle_errors(res.status, "list directory", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked ListContainer" trace="{res.status.trace}"') + for info in res.infos: + yield info + + def _set_lock_using_xattr(self, auth_token, resource: Resource, app_name: str, lock_id: Union[int, str]) -> None: + """" + Set a lock to a resource with the given value metadata and appname as holder + + :param resource: Resource to set lock to + :param app_name: Application name + :param lock_id: Metadata lock value + :return: None (Success) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute SetLock" {resource.get_file_ref_str()}" value="{lock_id}"') + try: + # The stat can raise a KeyError if the metadata (lock) attribute has not been set yet + # e.g. the file is not locked + _ = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + except KeyError: + expiration = int(time.time() + self._config.lock_expiration) + self.set_xattr(auth_token, resource, LOCK_ATTR_KEY, f"{app_name}!{lock_id}!{expiration}", None) + return + + def set_lock(self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[int, str]) -> None: + """ + Set a lock to a resource with the given value and appname as holder + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to set lock to + :param app_name: Application name + :param lock_id: encoded lock_id or metadata lock value if using xattr + :return: None (Success) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + self._set_lock_using_xattr(auth_token, resource, app_name, lock_id) + return + + lock = cs3spr.Lock( + type=cs3spr.LOCK_TYPE_WRITE, + lock_id=lock_id, + app_name=app_name, + expiration={"seconds": int(time.time() + + self._config.lock_expiration)}, + ) + req = cs3sp.SetLockRequest(ref=resource.ref, lock=lock) + res = self._gateway.SetLock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, set the lock using xattr + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + self._set_lock_using_xattr(auth_token, resource, app_name, lock_id) + return + + self._status_code_handler.handle_errors(res.status, "set lock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked SetLock" {resource.get_file_ref_str()}" ' + f'value="{lock_id} result="{res.status.trace}"') + + def _get_lock_using_xattr(self, auth_token: tuple, resource: Resource) -> dict: + """ + Get the lock metadata for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to get lock from + :return: dictionary (KEYS: lock_id, type, app_name, user, expiration) (Success) + :return: None (No lock set) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute getlock" {resource.get_file_ref_str()}"') + try: + currvalue = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + values = currvalue.split("!") + return { + "lock_id": values[1], + "type": 2, # LOCK_TYPE_WRITE, though this is advisory! + "app_name": values[0], + "user": {}, + "expiration": int(values[2]), + } + except KeyError: + return None + + def get_lock(self, auth_token: tuple, resource: Resource) -> Union[cs3spr.Lock, dict, None]: + """ + Get the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to get lock from + :return: dictionary (KEYS: lock_id, type, app_name, user, expiration) (Success) + :return: None (No lock set) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + return self._get_lock_using_xattr(auth_token, resource) + + req = cs3sp.GetLockRequest(ref=resource.ref) + res = self._gateway.GetLock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, get the lock using xattr + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + return self._get_lock_using_xattr(auth_token, resource) + + self._status_code_handler.handle_errors(res.status, "get lock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked GetLock" {resource.get_file_ref_str()}" result="{res.status.trace}"') + + # rebuild a dict corresponding to the internal JSON structure used by Reva + return { + "lock_id": res.lock.lock_id, + "type": res.lock.type, + "app_name": res.lock.app_name, + "user": ( + {"opaque_id": res.lock.user.opaque_id, "idp": res.lock.user.idp, "type": res.lock.user.type} + if res.lock.user.opaque_id + else {} + ), + "expiration": {"seconds": res.lock.expiration.seconds}, + } + + def _refresh_lock_using_xattr( + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[str, int], + existing_lock_id: Union[str, int] = None + ) -> None: + """ + Refresh the lock metadata for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to refresh lock for + :param app_name: Application name + :param lock_id: metadata value to set + :param existing_lock_id: existing metadata vlue + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute RefreshLock" {resource.get_file_ref_str()}" value="{lock_id}"') + try: + # The stat can raise a KeyError if the metadata (lock) attribute has not been set yet + # e.g. the file is not locked + currvalue = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + values = currvalue.split("!") + if values[0] == app_name and (not existing_lock_id or values[1] == existing_lock_id): + raise KeyError + self._log.info( + f'Failed precondition on RefreshLock" {resource.get_file_ref_str()}" appname="{app_name}" ' + f'value="{lock_id} previouslock="{currvalue}"' + ) + raise FileLockedException() + except KeyError: + expiration = int(time.time() + self._config.lock_expiration) + self.set_xattr(auth_token, resource, LOCK_ATTR_KEY, f"{app_name}!{lock_id}!{expiration}", None) + return + + def refresh_lock( + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[str, int], + existing_lock_id: Union[str, int] = None + ): + """ + Refresh the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to refresh lock for + :param app_name: Application name + :param lock_id: encoded lock_id or metadata lock value if using xattr + :param existing_lock_id: encoded lock_id or metadata lock value if using xattr + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + self._refresh_lock_using_xattr(auth_token, resource, app_name, lock_id, existing_lock_id) + return + + lock = cs3spr.Lock( + type=cs3spr.LOCK_TYPE_WRITE, + app_name=app_name, + lock_id=lock_id, + expiration={"seconds": int(time.time() + self._config.lock_expiration)}, + ) + req = cs3sp.RefreshLockRequest(ref=resource.ref, lock=lock, existing_lock_id=existing_lock_id) + res = self._gateway.RefreshLock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, refresh the lock using xattr + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + self._refresh_lock_using_xattr(auth_token, resource, app_name, lock_id, existing_lock_id) + return + self._status_code_handler.handle_errors(res.status, "refresh lock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked RefreshLock" {resource.get_file_ref_str()} result="{res.status.trace}" ' + f'value="{lock_id}" old_value="{existing_lock_id}"') + + def _unlock_using_xattr( + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[str, int] + ) -> None: + """ + Remove the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to unlock + :param app_name: Application name + :param lock_id: metadata lock value + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute unlock" {resource.get_file_ref_str()}" value="{lock_id}"') + try: + # The stat can raise a KeyError if the metadata (lock) attribute has not been set yet + # e.g. the file is not locked + currvalue = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + values = currvalue.split("!") + if values[0] == app_name and values[1] == lock_id: + raise KeyError + self._log.info( + f'Failed precondition on unlock" {resource.get_file_ref_str()}" appname="{app_name}" ' + f'value={lock_id} previouslock="{currvalue}"' + ) + raise FileLockedException() + except KeyError: + self.remove_xattr(auth_token, resource, LOCK_ATTR_KEY, None) + return + + def unlock(self, auth_token: tuple, resource: Resource, app_name, lock_id: Union[str, int]): + """ + Remove the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to unlock + :param app_name: app_name + :param lock_id: encoded lock_id or metadata lock value if using xattr + :return: None + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + self._unlock_using_xattr(auth_token, resource, app_name, lock_id) + return + + lock = cs3spr.Lock(type=cs3spr.LOCK_TYPE_WRITE, app_name=app_name, lock_id=lock_id) + req = cs3sp.UnlockRequest(ref=resource.ref, lock=lock) + res = self._gateway.Unlock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, set the lock using xattr and retry + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + self._unlock_using_xattr(auth_token, resource, app_name, lock_id) + return + + self._status_code_handler.handle_errors(res.status, "unlock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked Unlock" {resource.get_file_ref_str()} result="{res.status.trace}" ' + f'value="{lock_id}"') diff --git a/src/share.py b/cs3client/share.py similarity index 89% rename from src/share.py rename to cs3client/share.py index 81f1254..da401a3 100644 --- a/src/share.py +++ b/cs3client/share.py @@ -3,16 +3,10 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging -from auth import Auth -from cs3resource import Resource -from config import Config -from statuscodehandler import StatusCodeHandler - - import cs3.sharing.collaboration.v1beta1.collaboration_api_pb2 as cs3scapi from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub import cs3.sharing.collaboration.v1beta1.resources_pb2 as cs3scr @@ -24,6 +18,10 @@ import google.protobuf.field_mask_pb2 as field_masks import cs3.types.v1beta1.types_pb2 as cs3types +from .cs3resource import Resource +from .config import Config +from .statuscodehandler import StatusCodeHandler + class Share: """ @@ -35,7 +33,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -44,21 +41,26 @@ def __init__( :param config: Config object containing the configuration parameters. :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. :param status_code_handler: An instance of the StatusCodeHandler class. """ self._status_code_handler: StatusCodeHandler = status_code_handler self._gateway: GatewayAPIStub = gateway self._log: logging.Logger = log self._config: Config = config - self._auth: Auth = auth def create_share( - self, resource_info: cs3spr.ResourceInfo, opaque_id: str, idp: str, role: str, grantee_type: str + self, + auth_token: tuple, + resource_info: cs3spr.ResourceInfo, + opaque_id: str, + idp: str, + role: str, + grantee_type: str ) -> cs3scr.Share: """ Create a share for a resource to the user/group with the specified role, using their opaque id and idp. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource_info: Resource info, see file.stat (REQUIRED). :param opaque_id: Opaque group/user id, (REQUIRED). :param idp: Identity provider, (REQUIRED). @@ -74,7 +76,7 @@ def create_share( """ share_grant = Share._create_share_grant(opaque_id, idp, role, grantee_type) req = cs3scapi.CreateShareRequest(resource_info=resource_info, grant=share_grant) - res = self._gateway.CreateShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.CreateShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "create share", f'opaque_id="{opaque_id}" resource_id="{resource_info.id}"' ) @@ -85,11 +87,12 @@ def create_share( return res.share def list_existing_shares( - self, filter_list: list[cs3scr.Filter] = None, page_size: int = 0, page_token: str = None + self, auth_token: tuple, filter_list: list[cs3scr.Filter] = None, page_size: int = 0, page_token: str = None ) -> list[cs3scr.Share]: """ List shares based on a filter. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter object to filter the shares, see create_share_filter. :param page_size: Number of shares to return in a page, defaults to 0, server decides. :param page_token: Token to get to a specific page. @@ -98,7 +101,7 @@ def list_existing_shares( :raises: UnknownException (Unknown error) """ req = cs3scapi.ListSharesRequest(filters=filter_list, page_size=page_size, page_token=page_token) - res = self._gateway.ListExistingShares(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListExistingShares(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list existing shares", f'filter="{filter_list}"') self._log.debug( f'msg="Invoked ListExistingShares" filter="{filter_list}" res_count="{len(res.share_infos)}' @@ -106,11 +109,12 @@ def list_existing_shares( ) return (res.share_infos, res.next_page_token) - def get_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> cs3scr.Share: + def get_share(self, auth_token: tuple, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> cs3scr.Share: """ Get a share by its opaque id or share key (combination of resource_id, grantee and owner), one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :return: Share object. @@ -127,7 +131,7 @@ def get_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> else: raise ValueError("opaque_id or share_key is required") - res = self._gateway.GetShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.GetShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "get share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -137,11 +141,12 @@ def get_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> ) return res.share - def remove_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> None: + def remove_share(self, auth_token: tuple, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> None: """ Remove a share by its opaque id or share key (combination of resource_id, grantee and owner), one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :return: None @@ -157,7 +162,7 @@ def remove_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) req = cs3scapi.RemoveShareRequest(ref=cs3scr.ShareReference(key=share_key)) else: raise ValueError("opaque_id or share_key is required") - res = self._gateway.RemoveShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.RemoveShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "remove share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -168,11 +173,16 @@ def remove_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) return def update_share( - self, role: str, opaque_id: str = None, share_key: cs3scr.ShareKey = None, display_name: str = None + self, auth_token: tuple, + role: str, + opaque_id: str = None, + share_key: cs3scr.ShareKey = None, + display_name: str = None ) -> cs3scr.Share: """ Update a share by its opaque id. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id. (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :param role: Role to update the share, VIEWER or EDITOR (REQUIRED). @@ -195,7 +205,7 @@ def update_share( raise ValueError("opaque_id or share_key is required") req = cs3scapi.UpdateShareRequest(ref=ref, field=update) - res = self._gateway.UpdateShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.UpdateShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "update share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -206,12 +216,13 @@ def update_share( return res.share def list_received_existing_shares( - self, filter_list: list = None, page_size: int = 0, page_token: str = None + self, auth_token: tuple, filter_list: list = None, page_size: int = 0, page_token: str = None ) -> list: """ List received existing shares. NOTE: Filters for received shares are not yet implemented (14/08/2024) + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter object to filter the shares, see create_share_filter. :param page_size: Number of shares to return in a page, defaults to 0, server decides. :param page_token: Token to get to a specific page. @@ -220,7 +231,7 @@ def list_received_existing_shares( :raises: UnknownException (Unknown error) """ req = cs3scapi.ListReceivedSharesRequest(filters=filter_list, page_size=page_size, page_token=page_token) - res = self._gateway.ListExistingReceivedShares(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListExistingReceivedShares(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list received existing shares", f'filter="{filter_list}"') self._log.debug( f'msg="Invoked ListExistingReceivedShares" filter="{filter_list}" res_count="{len(res.share_infos)}"' @@ -228,11 +239,14 @@ def list_received_existing_shares( ) return (res.share_infos, res.next_page_token) - def get_received_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> cs3scr.ReceivedShare: + def get_received_share( + self, auth_token: tuple, opaque_id: str = None, share_key: cs3scr.ShareKey = None + ) -> cs3scr.ReceivedShare: """ Get a received share by its opaque id or share key (combination of resource_id, grantee and owner), one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id. (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :return: ReceivedShare object. @@ -248,7 +262,7 @@ def get_received_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = req = cs3scapi.GetReceivedShareRequest(ref=cs3scr.ShareReference(key=share_key)) else: raise ValueError("opaque_id or share_key is required") - res = self._gateway.GetReceivedShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.GetReceivedShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "get received share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -259,11 +273,12 @@ def get_received_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = return res.share def update_received_share( - self, received_share: cs3scr.ReceivedShare, state: str = "SHARE_STATE_ACCEPTED" + self, auth_token: tuple, received_share: cs3scr.ReceivedShare, state: str = "SHARE_STATE_ACCEPTED" ) -> cs3scr.ReceivedShare: """ Update the state of a received share (SHARE_STATE_ACCEPTED, SHARE_STATE_ACCEPTED, SHARE_STATE_REJECTED). + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param recieved_share: ReceivedShare object. :param state: Share state to update to, defaults to SHARE_STATE_ACCEPTED, (REQUIRED). :return: Updated ReceivedShare object. @@ -283,7 +298,7 @@ def update_received_share( ), update_mask=field_masks.FieldMask(paths=["state"]), ) - res = self._gateway.UpdateReceivedShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.UpdateReceivedShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "update received share", f'opaque_id="{received_share.share.id.opaque_id}"' ) @@ -295,6 +310,7 @@ def update_received_share( def create_public_share( self, + auth_token: tuple, resource_info: cs3spr.ResourceInfo, role: str, password: str = None, @@ -307,6 +323,7 @@ def create_public_share( """ Create a public share. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource_info: Resource info, see file.stat (REQUIRED). :param role: Role to assign to the grantee, VIEWER or EDITOR (REQUIRED) :param password: Password to access the share. @@ -335,17 +352,18 @@ def create_public_share( notify_uploads_extra_recipients=notify_uploads_extra_recipients, ) - res = self._gateway.CreatePublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.CreatePublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "create public share", f'resource_id="{resource_info.id}"') self._log.debug(f'msg="Invoked CreatePublicShare" resource_id="{resource_info.id}" trace="{res.status.trace}"') return res.share def list_existing_public_shares( - self, filter_list: list = None, page_size: int = 0, page_token: str = None, sign: bool = None + self, auth_token: tuple, filter_list: list = None, page_size: int = 0, page_token: str = None, sign: bool = None ) -> list: """ List existing public shares. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter object to filter the shares, see create_public_share_filter. :param page_size: Number of shares to return in a page, defaults to 0 and then the server decides. :param page_token: Token to get to a specific page. @@ -358,7 +376,7 @@ def list_existing_public_shares( req = cs3slapi.ListPublicSharesRequest( filters=filter_list, page_size=page_size, page_token=page_token, sign=sign ) - res = self._gateway.ListExistingPublicShares(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListExistingPublicShares(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list existing public shares", f'filter="{filter_list}"') self._log.debug( f'msg="Invoked ListExistingPublicShares" filter="{filter_list}" res_count="{len(res.share_infos)}" ' @@ -366,10 +384,13 @@ def list_existing_public_shares( ) return (res.share_infos, res.next_page_token) - def get_public_share(self, opaque_id: str = None, token: str = None, sign: bool = False) -> cs3slr.PublicShare: + def get_public_share( + self, auth_token: tuple, opaque_id: str = None, token: str = None, sign: bool = False + ) -> cs3slr.PublicShare: """ Get a public share by its opaque id or token, one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id (SEMI-OPTIONAL). :param share_token: Share token (SEMI-OPTIONAL). :param sign: if the signature should be included in the share. @@ -387,7 +408,7 @@ def get_public_share(self, opaque_id: str = None, token: str = None, sign: bool else: raise ValueError("token or opaque_id is required") req = cs3slapi.GetPublicShareRequest(ref=ref, sign=sign) - res = self._gateway.GetPublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.GetPublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "get public share", f'opaque_id/token="{opaque_id if opaque_id else token}"' ) @@ -399,6 +420,7 @@ def get_public_share(self, opaque_id: str = None, token: str = None, sign: bool def update_public_share( self, + auth_token: tuple, type: str, role: str, opaque_id: str = None, @@ -415,6 +437,7 @@ def update_public_share( however, other parameters are optional. Note that only the type of update specified will be applied. The role will only change if type is TYPE_PERMISSIONS. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param type: Type of update to perform TYPE_PERMISSIONS, TYPE_PASSWORD, TYPE_EXPIRATION, TYPE_DISPLAYNAME, TYPE_DESCRIPTION, TYPE_NOTIFYUPLOADS, TYPE_NOTIFYUPLOADSEXTRARECIPIENTS (REQUIRED). :param role: Role to assign to the grantee, VIEWER or EDITOR (REQUIRED). @@ -451,7 +474,7 @@ def update_public_share( password=password, ) req = cs3slapi.UpdatePublicShareRequest(ref=ref, update=update) - res = self._gateway.UpdatePublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.UpdatePublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "update public share", @@ -463,10 +486,11 @@ def update_public_share( ) return res.share - def remove_public_share(self, token: str = None, opaque_id: str = None) -> None: + def remove_public_share(self, auth_token: tuple, token: str = None, opaque_id: str = None) -> None: """ Remove a public share by its token or opaque id, one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param token: Share token (SEMI-OPTIONAL). :param opaque_id: Opaque share id (SEMI-OPTIONAL). :return: None @@ -481,7 +505,7 @@ def remove_public_share(self, token: str = None, opaque_id: str = None) -> None: raise ValueError("token or opaque_id is required") req = cs3slapi.RemovePublicShareRequest(ref=ref) - res = self._gateway.RemovePublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.RemovePublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "remove public share", f'opaque_id/token="{opaque_id if opaque_id else token}"' ) diff --git a/src/statuscodehandler.py b/cs3client/statuscodehandler.py similarity index 60% rename from src/statuscodehandler.py rename to cs3client/statuscodehandler.py index 2a63907..c468212 100644 --- a/src/statuscodehandler.py +++ b/cs3client/statuscodehandler.py @@ -3,73 +3,82 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -from exceptions.exceptions import AuthenticationException, NotFoundException, \ - UnknownException, AlreadyExistsException, FileLockedException, UnimplementedException import logging -from config import Config import cs3.rpc.v1beta1.code_pb2 as cs3code import cs3.rpc.v1beta1.status_pb2 as cs3status +from .exceptions.exceptions import AuthenticationException, NotFoundException, \ + UnknownException, AlreadyExistsException, FileLockedException, UnimplementedException +from .config import Config + class StatusCodeHandler: def __init__(self, log: logging.Logger, config: Config) -> None: self._log = log self._config = config - def _log_not_found_info(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_not_found_info(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.info( - f'msg="Not found on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'msg="Not found on {operation}" {msg + " " if msg else ""} ' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_authentication_error(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_authentication_error( + self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None + ) -> None: self._log.error( f'msg="Authentication failed on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_unknown_error(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_unknown_error(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.error( f'msg="Failed to {operation}, unknown error" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_precondition_info(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_precondition_info( + self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None + ) -> None: self._log.info( f'msg="Failed precondition on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_already_exists(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_already_exists(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.info( f'msg="Already exists on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_unimplemented(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_unimplemented(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.info( f'msg="Invoked {operation} on unimplemented feature" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status_msg}"' ) def handle_errors(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + if status.code == cs3code.CODE_OK: return + # status.message.replace('"', "'") is not allowed inside f strings python<3.12 + status_message = status.message.replace('"', "'") + if status.code == cs3code.CODE_FAILED_PRECONDITION or status.code == cs3code.CODE_ABORTED: - self._log_precondition_info(status, operation, msg) + self._log_precondition_info(status, operation, status_message, msg) raise FileLockedException(f'Failed precondition: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_ALREADY_EXISTS: - self._log_already_exists(status, operation, msg) + self._log_already_exists(status, operation, status_message, msg) raise AlreadyExistsException(f'Resource already exists: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_UNIMPLEMENTED: @@ -77,11 +86,11 @@ def handle_errors(self, status: cs3status.Status, operation: str, msg: str = Non raise UnimplementedException(f'Unimplemented feature: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_NOT_FOUND: - self._log_not_found_info(status, operation, msg) + self._log_not_found_info(status, operation, status_message, msg) raise NotFoundException(f'Not found: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_UNAUTHENTICATED: - self._log_authentication_error(status, operation, msg) + self._log_authentication_error(status, operation, status_message, msg) raise AuthenticationException(f'Operation not permitted: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code != cs3code.CODE_OK: @@ -89,8 +98,8 @@ def handle_errors(self, status: cs3status.Status, operation: str, msg: str = Non self._log.info(f'msg="Invoked {operation} on missing file" ') raise NotFoundException( message=f'No such file or directory: operation="{operation}" ' - f'status_code="{status.code}" message="{status.message}"' + f'status_code="{status.code}" message="{status.message}"' ) - self._log_unknown_error(status, operation, msg) + self._log_unknown_error(status, operation, status_message, msg) raise UnknownException(f'Unknown Error: operation="{operation}" status_code="{status.code}" ' f'message="{status.message}"') diff --git a/src/user.py b/cs3client/user.py similarity index 92% rename from src/user.py rename to cs3client/user.py index 2d64d4b..3c4b03d 100644 --- a/src/user.py +++ b/cs3client/user.py @@ -3,16 +3,16 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging -from auth import Auth -from config import Config import cs3.identity.user.v1beta1.resources_pb2 as cs3iur import cs3.identity.user.v1beta1.user_api_pb2 as cs3iu from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub -from statuscodehandler import StatusCodeHandler + +from .config import Config +from .statuscodehandler import StatusCodeHandler class User: @@ -25,7 +25,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -35,7 +34,6 @@ def __init__( :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. :param auth: An instance of the auth class. """ - self._auth: Auth = auth self._log: logging.Logger = log self._gateway: GatewayAPIStub = gateway self._config: Config = config @@ -92,10 +90,11 @@ def get_user_groups(self, idp, opaque_id) -> list[str]: self._log.debug(f'msg="Invoked GetUserGroups" opaque_id="{opaque_id}" trace="{res.status.trace}"') return res.groups - def find_users(self, filter) -> list[cs3iur.User]: + def find_users(self, auth_token: tuple, filter) -> list[cs3iur.User]: """ Find a user based on a filter. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter to search for. :return: a list of user(s). :raises: NotFoundException (User not found) @@ -103,7 +102,7 @@ def find_users(self, filter) -> list[cs3iur.User]: :raises: UnknownException (Unknown error) """ req = cs3iu.FindUsersRequest(filter=filter, skip_fetching_user_groups=True) - res = self._gateway.FindUsers(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.FindUsers(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "find users") self._log.debug(f'msg="Invoked FindUsers" filter="{filter}" trace="{res.status.trace}"') return res.users diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index eb02f0b..0000000 Binary files a/docs/.DS_Store and /dev/null differ diff --git a/examples/app_api_example.py b/examples/app_api_example.py index 202754e..21999ff 100644 --- a/examples/app_api_example.py +++ b/examples/app_api_example.py @@ -6,35 +6,43 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging import configparser -from cs3client import CS3Client -from cs3resource import Resource +from cs3client.cs3resource import Resource +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) - client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") - -print(client.auth.get_token()) +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) # list_app_providers -res = client.app.list_app_providers() +res = client.app.list_app_providers(auth.get_token()) if res is not None: print(res) # open_in_app -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/collabora.odt") -res = client.app.open_in_app(resource) +resource = Resource(abs_path="/eos/user/r/rwelande/collabora.odt") +res = client.app.open_in_app(auth.get_token(), resource) if res is not None: print(res) diff --git a/examples/auth_example.py b/examples/auth_example.py new file mode 100644 index 0000000..926c5c2 --- /dev/null +++ b/examples/auth_example.py @@ -0,0 +1,36 @@ +""" +auth_example.py + +Example script to demonstrate the usage of the app API in the CS3Client class. +note that these are examples, and is not meant to be run as a script. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 30/08/2024 +""" + +import logging +import configparser +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth + +config = configparser.ConfigParser() +with open("default.conf") as fdef: + config.read_file(fdef) +log = logging.getLogger(__name__) + +client = CS3Client(config, "cs3client", log) +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) diff --git a/examples/checkpoints_api_example.py b/examples/checkpoints_api_example.py index db4c094..5fa03d8 100644 --- a/examples/checkpoints_api_example.py +++ b/examples/checkpoints_api_example.py @@ -6,36 +6,47 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging import configparser -from cs3client import CS3Client -from cs3resource import Resource +from cs3client.cs3resource import Resource +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None -markdown_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/test.md") +markdown_resource = Resource(abs_path="/eos/user/r/rwelande/test.md") -res = client.checkpoint.list_file_versions(markdown_resource) +res = client.checkpoint.list_file_versions(auth.get_token(), markdown_resource) if res is not None: for ver in res: print(ver) -res = client.checkpoint.restore_file_version(markdown_resource, "1722936250.0569fa2f") +res = client.checkpoint.restore_file_version(auth.get_token(), markdown_resource, "1722936250.0569fa2f") if res is not None: for ver in res: print(ver) diff --git a/examples/default.conf b/examples/default.conf index e5be292..b8d0c68 100644 --- a/examples/default.conf +++ b/examples/default.conf @@ -33,6 +33,8 @@ ssl_ca_cert = test_ca_cert auth_client_id = einstein # Optional, defaults to basic auth_login_type = basic +# Optional (Can also be set after instantiating the Auth object) +auth_client_secret = relativity # For the future lock implementation diff --git a/examples/file_api_example.py b/examples/file_api_example.py index 468449b..fb25bed 100644 --- a/examples/file_api_example.py +++ b/examples/file_api_example.py @@ -13,76 +13,84 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 01/08/2024 +Last updated: 30/08/2024 """ import logging import configparser -from cs3client import CS3Client -from cs3resource import Resource +from cs3client.cs3client import CS3Client +from cs3client.cs3resource import Resource +from cs3client.auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -client.auth.set_token("") -# OR -# client.auth.set_client_secret("") - -# Authentication -print(client.auth.get_token()) +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None # mkdir for i in range(1, 4): - directory_resource = Resource.from_file_ref_and_endpoint(f"/eos/user/r/rwelande/test_directory{i}") - res = client.file.make_dir(directory_resource) + directory_resource = Resource(abs_path=f"/eos/user/r/rwelande/test_directory{i}") + res = client.file.make_dir(auth.get_token(), directory_resource) if res is not None: print(res) # touchfile -touch_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/touch_file.txt") -text_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") -res = client.file.touch_file(touch_resource) -res = client.file.touch_file(text_resource) +touch_resource = Resource(abs_path="/eos/user/r/rwelande/touch_file.txt") +text_resource = Resource(abs_path="/eos/user/r/rwelande/text_file.txt") +res = client.file.touch_file(auth.get_token(), touch_resource) +res = client.file.touch_file(auth.get_token(), text_resource) if res is not None: print(res) # setxattr -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") -res = client.file.set_xattr(resource, "iop.wopi.lastwritetime", str(1720696124)) +resource = Resource(abs_path="/eos/user/r/rwelande/text_file.txt") +res = client.file.set_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime", str(1720696124)) if res is not None: print(res) # rmxattr -res = client.file.remove_xattr(resource, "iop.wopi.lastwritetime") +res = client.file.remove_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime") if res is not None: print(res) # stat -res = client.file.stat(text_resource) +res = client.file.stat(auth.get_token(), text_resource) if res is not None: print(res) # removefile -res = client.file.remove_file(touch_resource) +res = client.file.remove_file(auth.get_token(), touch_resource) if res is not None: print(res) -res = client.file.touch_file(touch_resource) +res = client.file.touch_file(auth.get_token(), touch_resource) # rename -rename_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/rename_file.txt") -res = client.file.rename_file(resource, rename_resource) +rename_resource = Resource(abs_path="/eos/user/r/rwelande/rename_file.txt") +res = client.file.rename_file(auth.get_token(), resource, rename_resource) if res is not None: print(res) @@ -90,20 +98,20 @@ # writefile content = b"Hello World" size = len(content) -res = client.file.write_file(rename_resource, content, size) +res = client.file.write_file(auth.get_token(), rename_resource, content, size) if res is not None: print(res) # rmdir (same as deletefile) -res = client.file.remove_file(directory_resource) +res = client.file.remove_file(auth.get_token(), directory_resource) if res is not None: print(res) # listdir -list_directory_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande") -res = client.file.list_dir(list_directory_resource) +list_directory_resource = Resource(abs_path="/eos/user/r/rwelande") +res = client.file.list_dir(auth.get_token(), list_directory_resource) first_item = next(res, None) if first_item is not None: @@ -114,7 +122,7 @@ print("empty response") # readfile -file_res = client.file.read_file(rename_resource) +file_res = client.file.read_file(auth.get_token(), rename_resource) content = b"" try: for chunk in file_res: diff --git a/examples/lock_example.py b/examples/lock_example.py new file mode 100644 index 0000000..6438af2 --- /dev/null +++ b/examples/lock_example.py @@ -0,0 +1,65 @@ +""" +lock_example.py + +Example script to demonstrate the usage of the app API in the CS3Client class. +note that these are examples, and is not meant to be run as a script. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 30/08/2024 +""" + +import logging +import configparser +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth +from cs3client.cs3resource import Resource + + +config = configparser.ConfigParser() +with open("default.conf") as fdef: + config.read_file(fdef) +log = logging.getLogger(__name__) + +client = CS3Client(config, "cs3client", log) +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) + +resource = Resource(abs_path="/eos/user/r/rwelande/lock_test.txt") + +# Set lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id="some_lock") + +# Get lock +res = client.file.get_lock(auth_token, resource) +if res is not None: + lock_id = res["lock_id"] + print(res) + +# Unlock +res = client.file.unlock(auth_token, resource, app_name="a", lock_id=lock_id) + +# Refresh lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id="some_lock") +res = client.file.refresh_lock( + auth_token, resource, app_name="a", lock_id="new_lock", existing_lock_id=lock_id +) + +if res is not None: + print(res) + +res = client.file.get_lock(auth_token, resource) +if res is not None: + print(res) diff --git a/examples/shares_api_example.py b/examples/shares_api_example.py index 5a42c8d..f78a1fe 100644 --- a/examples/shares_api_example.py +++ b/examples/shares_api_example.py @@ -6,43 +6,55 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging import configparser -from cs3client import CS3Client -from cs3resource import Resource +from cs3client.cs3client import CS3Client +from cs3client.cs3resource import Resource +from cs3client.auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") - -# Authentication -print(client.auth.get_token()) +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None # Create share # -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text.txt") -resource_info = client.file.stat(resource) +resource = Resource(abs_path="/eos/user/r/rwelande/text.txt") +resource_info = client.file.stat(auth.get_token(), resource) # VIEWER user = client.user.get_user_by_claim("mail", "diogo.castro@cern.ch") -res = client.share.create_share(resource_info, user.id.opaque_id, user.id.idp, "VIEWER", "USER") +res = client.share.create_share( + auth.get_token(), resource_info, user.id.opaque_id, user.id.idp, "VIEWER", "USER" +) if res is not None: print(res) # EDITOR user = client.user.get_user_by_claim("username", "lopresti") -res = client.share.create_share(resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER") +res = client.share.create_share( + auth.get_token(), resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER" +) if res is not None: print(res) @@ -54,26 +66,26 @@ filter_list.append(filter) filter = client.share.create_share_filter(share_state="SHARE_STATE_PENDING", filter_type="TYPE_STATE") filter_list.append(filter) -res, _ = client.share.list_existing_shares() +res, _ = client.share.list_existing_shares(auth.get_token(), filter_list=filter_list) if res is not None: for share_info in res: print(share_info.share) # Get share # share_id = "58" -res = client.share.get_share(opaque_id=share_id) +res = client.share.get_share(auth.get_token(), opaque_id=share_id) if res is not None: print(res) # update share # share_id = "58" -res = client.share.update_share(opaque_id=share_id, role="VIEWER") +res = client.share.update_share(auth.get_token(), opaque_id=share_id, role="VIEWER") if res is not None: print(res) # remove share # share_id = "58" -res = client.share.remove_share(opaque_id=share_id) +res = client.share.remove_share(auth.get_token(), opaque_id=share_id) if res is not None: print(res) @@ -87,7 +99,7 @@ filter_list.append(filter) # NOTE: filters for received shares are not implemented (14/08/2024), therefore it is left out -res, _ = client.share.list_received_existing_shares() +res, _ = client.share.list_received_existing_shares(auth.get_token()) if res is not None: for share_info in res: print(share_info.received_share) @@ -95,17 +107,19 @@ # get received share # share_id = "43" -received_share = client.share.get_received_share(opaque_id=share_id) +received_share = client.share.get_received_share(auth.get_token(), opaque_id=share_id) if received_share is not None: print(received_share) # update recieved share # -res = client.share.update_received_share(received_share=received_share, state="SHARE_STATE_ACCEPTED") +res = client.share.update_received_share( + auth.get_token(), received_share=received_share, state="SHARE_STATE_ACCEPTED" +) if res is not None: print(res) # create public share # -res = client.share.create_public_share(resource_info, role="VIEWER") +res = client.share.create_public_share(auth.get_token(), resource_info, role="VIEWER") if res is not None: print(res) @@ -116,7 +130,7 @@ filter = client.share.create_public_share_filter(resource_id=resource_info.id, filter_type="TYPE_RESOURCE_ID") filter_list.append(filter) print(filter_list) -res, _ = client.share.list_existing_public_shares(filter_list=filter_list) +res, _ = client.share.list_existing_public_shares(auth.get_token(), filter_list=filter_list) if res is not None: for share_info in res: print(share_info.share) @@ -125,16 +139,18 @@ share_id = "63" # OR token = "7FbP1EBXJQTqK0d" -res = client.share.get_public_share(opaque_id=share_id, sign=True) +res = client.share.get_public_share(auth.get_token(), opaque_id=share_id, sign=True) if res is not None: print(res) # update public share # -res = client.share.update_public_share(type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello") +res = client.share.update_public_share( + auth.get_token(), type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello" +) if res is not None: print(res) # remove public share # -res = client.share.remove_public_share(token=token) +res = client.share.remove_public_share(auth.get_token(), token=token) if res is not None: print(res) diff --git a/examples/user_api_example.py b/examples/user_api_example.py index c4a6973..7fdc105 100644 --- a/examples/user_api_example.py +++ b/examples/user_api_example.py @@ -6,29 +6,40 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 02/08/2024 +Last updated: 30/08/2024 """ import logging import configparser -from cs3client import CS3Client +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None # find_user -res = client.user.find_users("rwel") +res = client.user.find_users(client.auth.get_token(), "rwel") if res is not None: print(res) diff --git a/setup.py b/setup.py index 3b83556..2dd693a 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 26/07/2024 +Last updated: 30/08/2024 """ from setuptools import setup, find_packages @@ -20,12 +20,10 @@ name="cs3client", version=version, author="Rasmus Welander, Diogo Castro, Giuseppe Lo Presti", - package_dir={"": "src"}, - packages=find_packages(where="src"), - py_modules=["cs3client"], description="CS3 client for Python", long_description=long_description, long_description_content_type="text/markdown", + packages=find_packages(exclude=['tests*']), url="https://github.com/cs3org/cs3-python-client", install_requires=[ "grpcio>=1.47.0", @@ -37,4 +35,15 @@ "protobuf", "cryptography", ], + license="Apache 2.0", + classifiers=[ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Operating System :: OS Independent", + ], ) diff --git a/src/file.py b/src/file.py deleted file mode 100644 index 8abd9b6..0000000 --- a/src/file.py +++ /dev/null @@ -1,329 +0,0 @@ -""" -file.py - -Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. -Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 -""" - -import time -import logging -import http -import requests -import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr -import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3sp -from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub -import cs3.types.v1beta1.types_pb2 as types - -from config import Config -from typing import Generator -from exceptions.exceptions import AuthenticationException, FileLockedException -from cs3resource import Resource -from auth import Auth -from statuscodehandler import StatusCodeHandler - - -class File: - """ - File class to interact with the CS3 API. - """ - - def __init__( - self, config: Config, log: logging.Logger, gateway: GatewayAPIStub, auth: Auth, - status_code_handler: StatusCodeHandler - ) -> None: - """ - Initializes the File class with configuration, logger, auth, gateway stub, and status code handler. - - :param config: Config object containing the configuration parameters. - :param log: Logger instance for logging. - :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. - :param status_code_handler: An instance of the StatusCodeHandler class. - """ - self._auth: Auth = auth - self._config: Config = config - self._log: logging.Logger = log - self._gateway: GatewayAPIStub = gateway - self._status_code_handler: StatusCodeHandler = status_code_handler - - def stat(self, resource: Resource) -> cs3spr.ResourceInfo: - """ - Stat a file and return the ResourceInfo object. - - :param resource: resource to stat. - :return: cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo (success) - :raises: NotFoundException (File not found) - :raises: AuthenticationException (Authentication Failed) - :raises: UnknownException (Unknown Error) - - """ - tstart = time.time() - res = self._gateway.Stat(request=cs3sp.StatRequest(ref=resource.ref), metadata=[self._auth.get_token()]) - tend = time.time() - self._status_code_handler.handle_errors(res.status, "stat", resource.get_file_ref_str()) - self._log.info( - f'msg="Invoked Stat" fileref="{resource.ref}" {resource.get_file_ref_str()} trace="{res.status.trace}" ' - f'elapsedTimems="{(tend - tstart) * 1000:.1f}"' - ) - return res.info - - def set_xattr(self, resource: Resource, key: str, value: str) -> None: - """ - Set the extended attribute to for a resource. - - :param resource: resource that has the attribute. - :param key: attribute key. - :param value: value to set. - :return: None (Success) - :raises: FileLockedException (File is locked) - :raises: AuthenticationException (Authentication Failed) - :raises: UnknownException (Unknown error) - """ - md = cs3spr.ArbitraryMetadata() - md.metadata.update({key: value}) # pylint: disable=no-member - req = cs3sp.SetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata=md) - res = self._gateway.SetArbitraryMetadata(request=req, metadata=[self._auth.get_token()]) - # CS3 storages may refuse to set an xattr in case of lock mismatch: this is an overprotection, - # as the lock should concern the file's content, not its metadata, however we need to handle that - self._status_code_handler.handle_errors(res.status, "set extended attribute", resource.get_file_ref_str()) - self._log.debug(f'msg="Invoked setxattr" trace="{res.status.trace}"') - - def remove_xattr(self, resource: Resource, key: str) -> None: - """ - Remove the extended attribute . - - :param resource: cs3client resource. - :param key: key for attribute to remove. - :return: None (Success) - :raises: FileLockedException (File is locked) - :raises: AuthenticationException (Authentication failed) - :raises: UnknownException (Unknown error) - """ - req = cs3sp.UnsetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata_keys=[key]) - res = self._gateway.UnsetArbitraryMetadata(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "remove extended attribute", resource.get_file_ref_str()) - self._log.debug(f'msg="Invoked UnsetArbitraryMetaData" trace="{res.status.trace}"') - - def rename_file(self, resource: Resource, newresource: Resource) -> None: - """ - Rename/move resource to new resource. - - :param resource: Original resource. - :param newresource: New resource. - :return: None (Success) - :raises: NotFoundException (Original resource not found) - :raises: FileLockException (Resource is locked) - :raises: AuthenticationException (Authentication Failed) - :raises: UnknownException (Unknown Error) - """ - req = cs3sp.MoveRequest(source=resource.ref, destination=newresource.ref) - res = self._gateway.Move(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "rename file", resource.get_file_ref_str()) - self._log.debug(f'msg="Invoked Move" trace="{res.status.trace}"') - - def remove_file(self, resource: Resource) -> None: - """ - Remove a resource. - - :param resource: Resource to remove. - :return: None (Success) - :raises: AuthenticationException (Authentication Failed) - :raises: NotFoundException (Resource not found) - :raises: UnknownException (Unknown error) - """ - req = cs3sp.DeleteRequest(ref=resource.ref) - res = self._gateway.Delete(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "remove file", resource.get_file_ref_str()) - self._log.debug(f'msg="Invoked Delete" trace="{res.status.trace}"') - - def touch_file(self, resource: Resource) -> None: - """ - Create a resource. - - :param resource: Resource to create. - :return: None (Success) - :raises: FileLockedException (File is locked) - :raises: AuthenticationException (Authentication Failed) - :raises: UnknownException (Unknown error) - """ - req = cs3sp.TouchFileRequest( - ref=resource.ref, - opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode("0"))}), - ) - res = self._gateway.TouchFile(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "touch file", resource.get_file_ref_str()) - self._log.debug(f'msg="Invoked TouchFile" trace="{res.status.trace}"') - - def write_file(self, resource: Resource, content: str | bytes, size: int) -> None: - """ - Write a file using the given userid as access token. The entire content is written - and any pre-existing file is deleted (or moved to the previous version if supported), - writing a file with size 0 is equivalent to "touch file" and should be used if the - implementation does not support touchfile. - - :param resource: Resource to write content to. - :param content: content to write - :param size: size of content (optional) - :return: None (Success) - :raises: FileLockedException (File is locked), - :raises: AuthenticationException (Authentication failed) - :raises: UnknownException (Unknown error) - - """ - tstart = time.time() - # prepare endpoint - if size == -1: - if isinstance(content, str): - content = bytes(content, "UTF-8") - size = len(content) - req = cs3sp.InitiateFileUploadRequest( - ref=resource.ref, - opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode(str(size)))}), - ) - res = self._gateway.InitiateFileUpload(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "write file", resource.get_file_ref_str()) - tend = time.time() - self._log.debug( - f'msg="Invoked InitiateFileUpload" trace="{res.status.trace}" protocols="{res.protocols}"' - ) - - # Upload - try: - protocol = [p for p in res.protocols if p.protocol in ["simple", "spaces"]][0] - if self._config.tus_enabled: - headers = { - "Tus-Resumable": "1.0.0", - "File-Path": resource.file, - "File-Size": str(size), - "X-Reva-Transfer": protocol.token, - **dict([self._auth.get_token()]), - } - else: - headers = { - "Upload-Length": str(size), - "X-Reva-Transfer": protocol.token, - **dict([self._auth.get_token()]), - } - putres = requests.put( - url=protocol.upload_endpoint, - data=content, - headers=headers, - verify=self._config.ssl_verify, - timeout=self._config.http_timeout, - ) - except requests.exceptions.RequestException as e: - self._log.error(f'msg="Exception when uploading file to Reva" reason="{e}"') - raise IOError(e) from e - if putres.status_code == http.client.CONFLICT: - self._log.info( - f'msg="Got conflict on PUT, file is locked" reason="{putres.reason}" {resource.get_file_ref_str()}' - ) - raise FileLockedException(f"Lock mismatch or lock expired: {putres.reason}") - if putres.status_code == http.client.UNAUTHORIZED: - self._log.warning( - f'msg="Authentication failed on write" reason="{putres.reason}" {resource.get_file_ref_str()}' - ) - raise AuthenticationException(f"Operation not permitted: {putres.reason}") - if putres.status_code != http.client.OK: - if ( - size == 0 - ): # 0-byte file uploads may have been finalized after InitiateFileUploadRequest, let's assume it's OK - # Should use TouchFileRequest instead - self._log.info( - f'msg="0-byte file written successfully" {resource.get_file_ref_str()} ' - f' elapsedTimems="{(tend - tstart) * 1000:.1f}"' - ) - return - - self._log.error( - f'msg="Error uploading file" code="{putres.status_code}" reason="{putres.reason}"' - ) - raise IOError(putres.reason) - self._log.info( - f'msg="File written successfully" {resource.get_file_ref_str()} ' - f'elapsedTimems="{(tend - tstart) * 1000:.1f}"' - ) - - def read_file(self, resource: Resource) -> Generator[bytes, None, None]: - """ - Read a file. Note that the function is a generator, managed by the app server. - - :param resource: Resource to read. - :return: Generator[Bytes, None, None] (Success) - :raises: NotFoundException (Resource not found) - :raises: AuthenticationException (Authentication Failed) - :raises: UnknownException (Unknown Error) - """ - tstart = time.time() - - # prepare endpoint - req = cs3sp.InitiateFileDownloadRequest(ref=resource.ref) - res = self._gateway.InitiateFileDownload(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "read file", resource.get_file_ref_str()) - tend = time.time() - self._log.debug( - f'msg="Invoked InitiateFileDownload" trace="{res.status.trace}" protocols="{res.protocols}"' - ) - - # Download - try: - protocol = [p for p in res.protocols if p.protocol in ["simple", "spaces"]][0] - headers = {"X-Reva-Transfer": protocol.token, **dict([self._auth.get_token()])} - fileget = requests.get( - url=protocol.download_endpoint, - headers=headers, - verify=self._config.ssl_verify, - timeout=self._config.http_timeout, - stream=True, - ) - except requests.exceptions.RequestException as e: - self._log.error(f'msg="Exception when downloading file from Reva" reason="{e}"') - raise IOError(e) - data = fileget.iter_content(self._config.chunk_size) - if fileget.status_code != http.client.OK: - self._log.error( - f'msg="Error downloading file from Reva" code="{fileget.status_code}" ' - f'reason="{fileget.reason.replace('"', "'")}"' - ) - raise IOError(fileget.reason) - else: - self._log.info( - f'msg="File open for read" {resource.get_file_ref_str()} elapsedTimems="{(tend - tstart) * 1000:.1f}"' - ) - for chunk in data: - yield chunk - - def make_dir(self, resource: Resource) -> None: - """ - Create a directory. - - :param resource: Direcotry to create. - :return: None (Success) - :raises: FileLockedException (File is locked) - :raises: AuthenticationException (Authentication failed) - :raises: UnknownException (Unknown error) - """ - req = cs3sp.CreateContainerRequest(ref=resource.ref) - res = self._gateway.CreateContainer(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "make directory", resource.get_file_ref_str()) - self._log.debug(f'msg="Invoked CreateContainer" trace="{res.status.trace}"') - - def list_dir( - self, resource: Resource - ) -> Generator[cs3spr.ResourceInfo, None, None]: - """ - List the contents of a directory, note that the function is a generator. - - :param resource: the directory. - :return: Generator[cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo, None, None] (Success) - :raises: NotFoundException (Resrouce not found) - :raises: AuthenticationException (Authentication Failed) - :raises: UnknownException (Unknown error) - """ - req = cs3sp.ListContainerRequest(ref=resource.ref) - res = self._gateway.ListContainer(request=req, metadata=[self._auth.get_token()]) - self._status_code_handler.handle_errors(res.status, "list directory", resource.get_file_ref_str()) - self._log.debug(f'msg="Invoked ListContainer" trace="{res.status.trace}"') - for info in res.infos: - yield info diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures.py b/tests/fixtures.py index 075f03e..b70b26e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -5,28 +5,26 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch from configparser import ConfigParser -import cs3.rpc.v1beta1.code_pb2 as cs3code import base64 import json +import cs3.rpc.v1beta1.code_pb2 as cs3code -sys.path.append("src/") -from cs3client import CS3Client # noqa: E402 -from file import File # noqa: E402 -from auth import Auth # noqa: E402 -from user import User # noqa: E402 -from statuscodehandler import StatusCodeHandler # noqa: E402 -from share import Share # noqa: E402 -from app import App # noqa: E402 -from checkpoint import Checkpoint # noqa: E402 -from config import Config # noqa: E402 +from cs3client.cs3client import CS3Client +from cs3client.file import File +from cs3client.auth import Auth +from cs3client.user import User +from cs3client.statuscodehandler import StatusCodeHandler +from cs3client.share import Share +from cs3client.app import App +from cs3client.checkpoint import Checkpoint +from cs3client.config import Config @pytest.fixture @@ -89,7 +87,6 @@ def mock_status_code_handler(mock_logger, mock_config): def mock_gateway(mock_gateway_stub_class): mock_gateway_stub = Mock() mock_gateway_stub_class.return_value = mock_gateway_stub - # Set up mock response for Authenticate method mocked_token = create_mock_jwt() mock_authenticate_response = Mock() mock_authenticate_response.status.code = cs3code.CODE_OK @@ -99,24 +96,15 @@ def mock_gateway(mock_gateway_stub_class): return mock_gateway_stub -# All the parameters are inferred by pytest from existing fixtures -@pytest.fixture -def mock_authentication(mock_gateway, mock_config, mock_logger): - # Set up mock response for Authenticate method - mock_authentication = Auth(Config(mock_config, "cs3client"), mock_logger, mock_gateway) - mock_authentication.set_client_secret("test") - return mock_authentication - - # Here the order of patches correspond to the parameters of the function # (patches are applied from the bottom up) # and the last two parameters are inferred by pytest from existing fixtures @pytest.fixture -@patch("cs3client.grpc.secure_channel", autospec=True) -@patch("cs3client.grpc.channel_ready_future", autospec=True) -@patch("cs3client.grpc.insecure_channel", autospec=True) -@patch("cs3client.cs3gw_grpc.GatewayAPIStub", autospec=True) -@patch("cs3client.grpc.ssl_channel_credentials", autospec=True) +@patch("cs3client.cs3client.grpc.secure_channel", autospec=True) +@patch("cs3client.cs3client.grpc.channel_ready_future", autospec=True) +@patch("cs3client.cs3client.grpc.insecure_channel", autospec=True) +@patch("cs3client.cs3client.cs3gw_grpc.GatewayAPIStub", autospec=True) +@patch("cs3client.cs3client.grpc.ssl_channel_credentials", autospec=True) def cs3_client_secure( mock_ssl_channel_credentials, mock_gateway_stub_class, @@ -129,7 +117,6 @@ def cs3_client_secure( # Create CS3Client instance client = CS3Client(mock_config, "cs3client", mock_logger) - client.auth.set_client_secret("test") assert mock_secure_channel.called assert mock_channel_ready_future.called @@ -143,11 +130,11 @@ def cs3_client_secure( # (patches are applied from the bottom up) # and the last two parameters are inferred by pytest from existing fixtures @pytest.fixture -@patch("cs3client.grpc.secure_channel") -@patch("cs3client.grpc.insecure_channel") -@patch("cs3client.grpc.channel_ready_future") -@patch("cs3client.cs3gw_grpc.GatewayAPIStub") -@patch("cs3client.grpc.ssl_channel_credentials") +@patch("cs3client.cs3client.grpc.secure_channel") +@patch("cs3client.cs3client.grpc.insecure_channel") +@patch("cs3client.cs3client.grpc.channel_ready_future") +@patch("cs3client.cs3client.cs3gw_grpc.GatewayAPIStub") +@patch("cs3client.cs3client.grpc.ssl_channel_credentials") def cs3_client_insecure( mock_ssl_channel_credentials, mock_gateway_stub_class, @@ -161,7 +148,6 @@ def cs3_client_insecure( # Create CS3Client instance client = CS3Client(mock_config, "cs3client", mock_logger) - client.auth.set_client_secret("test") assert mock_insecure_channel.called assert mock_channel_ready_future.called @@ -171,28 +157,36 @@ def cs3_client_insecure( @pytest.fixture -def app_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def auth_instance(cs3_client_insecure): + # Set up mock response for Authenticate method + auth = Auth(cs3_client_insecure) + auth.set_client_secret("test") + return auth + + +# All the parameters are inferred by pytest from existing fixtures +@pytest.fixture +def app_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): app = App( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return app @pytest.fixture -def checkpoint_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def checkpoint_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): checkpoint = Checkpoint( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return checkpoint @pytest.fixture -def share_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def share_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): share = Share( Config(mock_config, "cs3client"), mock_logger, mock_gateway, - mock_authentication, mock_status_code_handler, ) return share @@ -200,16 +194,16 @@ def share_instance(mock_authentication, mock_gateway, mock_config, mock_logger, # All parameters are inferred by pytest from existing fixtures @pytest.fixture -def file_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def file_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): file = File( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return file @pytest.fixture -def user_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def user_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): user = User( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return user diff --git a/tests/test_app.py b/tests/test_app.py index 8e3cf93..0233797 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -5,34 +5,28 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys import cs3.rpc.v1beta1.code_pb2 as cs3code from unittest.mock import Mock, patch import pytest -sys.path.append("src/") - -from exceptions.exceptions import ( # noqa: E402 +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) -from cs3resource import Resource # noqa: E402 - -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from cs3client.cs3resource import Resource +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, app_instance, mock_status_code_handler, ) # Test cases for the App class -# Test cases for the App class `list_app_providers` method using parameterized tests @pytest.mark.parametrize( @@ -50,13 +44,14 @@ def test_list_app_providers( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.providers = providers + auth_token = ('x-access-token', "some_token") with patch.object(app_instance._gateway, "ListAppProviders", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - app_instance.list_app_providers() + app_instance.list_app_providers(auth_token) else: - result = app_instance.list_app_providers() + result = app_instance.list_app_providers(auth_token) assert result == providers @@ -80,11 +75,12 @@ def test_open_in_app( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.OpenInAppURL = open_in_app_url + auth_token = ('x-access-token', "some_token") with patch.object(app_instance._gateway, "OpenInApp", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - app_instance.open_in_app(resource, view_mode, app) + app_instance.open_in_app(auth_token, resource, view_mode, app) else: - result = app_instance.open_in_app(resource, view_mode, app) + result = app_instance.open_in_app(auth_token, resource, view_mode, app) assert result == open_in_app_url diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 20c679d..d005965 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -5,34 +5,27 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys from unittest.mock import Mock, patch import pytest import cs3.rpc.v1beta1.code_pb2 as cs3code -sys.path.append("src/") - -from exceptions.exceptions import ( # noqa: E402 +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) - -from cs3resource import Resource # noqa: E402 - -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from cs3client.cs3resource import Resource +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, checkpoint_instance, mock_status_code_handler, ) - # Test cases for the Checkpoint class @@ -56,13 +49,14 @@ def test_list_file_versions( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.versions = versions + auth_token = ('x-access-token', "some_token") with patch.object(checkpoint_instance._gateway, "ListFileVersions", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - checkpoint_instance.list_file_versions(resource, page_token, page_size) + checkpoint_instance.list_file_versions(auth_token, resource, page_token, page_size) else: - result = checkpoint_instance.list_file_versions(resource, page_token, page_size) + result = checkpoint_instance.list_file_versions(auth_token, resource, page_token, page_size) assert result == versions @@ -85,11 +79,12 @@ def test_restore_file_version( mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(checkpoint_instance._gateway, "RestoreFileVersion", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - checkpoint_instance.restore_file_version(resource, version_key, lock_id) + checkpoint_instance.restore_file_version(auth_token, resource, version_key, lock_id) else: - result = checkpoint_instance.restore_file_version(resource, version_key, lock_id) + result = checkpoint_instance.restore_file_version(auth_token, resource, version_key, lock_id) assert result is None diff --git a/tests/test_cs3client.py b/tests/test_cs3client.py index 2aa169c..e4e6872 100644 --- a/tests/test_cs3client.py +++ b/tests/test_cs3client.py @@ -5,14 +5,10 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 26/07/2024 +Last updated: 30/08/2024 """ -import sys - -sys.path.append("src/") - -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) cs3_client_insecure, cs3_client_secure, mock_config, @@ -21,6 +17,8 @@ create_mock_jwt, ) +# Test cases for the cs3client class. + def test_cs3client_initialization_secure(cs3_client_secure): # noqa: F811 (not a redefinition) client = cs3_client_secure @@ -45,17 +43,10 @@ def test_cs3client_initialization_secure(cs3_client_secure): # noqa: F811 (not # Make sure the gRPC channel is correctly created assert client.channel is not None assert client._gateway is not None - assert client.auth is not None assert client.file is not None - # Make sure auth objects are correctly set - assert client.auth._gateway is not None - assert client.auth._config is not None - assert client.auth._log is not None - # Make sure file objects are correctly set assert client.file._gateway is not None - assert client.file._auth is not None assert client.file._config is not None assert client.file._log is not None @@ -83,16 +74,9 @@ def test_cs3client_initialization_insecure(cs3_client_insecure): # noqa: F811 ( # Make sure the gRPC channel is correctly created assert client.channel is not None assert client._gateway is not None - assert client.auth is not None assert client.file is not None - # Make sure auth objects are correctly set - assert client.auth._gateway is not None - assert client.auth._config is not None - assert client.auth._log is not None - # Make sure file objects are correctly set assert client.file._gateway is not None - assert client.file._auth is not None assert client.file._config is not None assert client.file._log is not None diff --git a/tests/test_file.py b/tests/test_file.py index 24d0a94..7e83afa 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -5,32 +5,30 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch import cs3.rpc.v1beta1.code_pb2 as cs3code -sys.path.append("src/") - -from cs3resource import Resource # noqa: E402 -from exceptions.exceptions import ( # noqa: E402 +from cs3client.cs3resource import Resource +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, FileLockedException, UnknownException, ) -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, file_instance, mock_status_code_handler, ) +# Test cases for the file class. + @pytest.mark.parametrize( "status_code, status_message, expected_exception, expected_result", @@ -48,15 +46,16 @@ def test_stat( mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") if status_code == cs3code.CODE_OK: mock_response.info = status_message with patch.object(file_instance._gateway, "Stat", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.stat(resource) + file_instance.stat(auth_token, resource) else: - result = file_instance.stat(resource) + result = file_instance.stat(auth_token, resource) assert result == expected_result @@ -76,13 +75,14 @@ def test_set_xattr(file_instance, status_code, status_message, expected_exceptio mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "SetArbitraryMetadata", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.set_xattr(resource, key, value) + file_instance.set_xattr(auth_token, resource, key, value) else: - file_instance.set_xattr(resource, key, value) + file_instance.set_xattr(auth_token, resource, key, value) @pytest.mark.parametrize( @@ -103,13 +103,14 @@ def test_remove_xattr( mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "UnsetArbitraryMetadata", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.remove_xattr(resource, key) + file_instance.remove_xattr(auth_token, resource, key) else: - file_instance.remove_xattr(resource, key) + file_instance.remove_xattr(auth_token, resource, key) @pytest.mark.parametrize( @@ -129,13 +130,14 @@ def test_rename_file(file_instance, status_code, status_message, expected_except mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "Move", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.rename_file(resource, newresource) + file_instance.rename_file(auth_token, resource, newresource) else: - file_instance.rename_file(resource, newresource) + file_instance.rename_file(auth_token, resource, newresource) @pytest.mark.parametrize( @@ -152,13 +154,14 @@ def test_remove_file(file_instance, status_code, status_message, expected_except mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "Delete", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.remove_file(resource) + file_instance.remove_file(auth_token, resource) else: - file_instance.remove_file(resource) + file_instance.remove_file(auth_token, resource) @pytest.mark.parametrize( @@ -175,13 +178,14 @@ def test_touch_file(file_instance, status_code, status_message, expected_excepti mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "TouchFile", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.touch_file(resource) + file_instance.touch_file(auth_token, resource) else: - file_instance.touch_file(resource) + file_instance.touch_file(auth_token, resource) @pytest.mark.parametrize( @@ -208,6 +212,7 @@ def test_write_file( mock_upload_response.status.code = status_code mock_upload_response.status.message = status_message mock_upload_response.protocols = [Mock(protocol="simple", upload_endpoint="http://example.com", token="token")] + auth_token = ('x-access-token', "some_token") mock_put_response = Mock() mock_put_response.status_code = put_response_status @@ -215,10 +220,10 @@ def test_write_file( with patch.object(file_instance._gateway, "InitiateFileUpload", return_value=mock_upload_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.write_file(resource, content, size) + file_instance.write_file(auth_token, resource, content, size) else: with patch("requests.put", return_value=mock_put_response): - file_instance.write_file(resource, content, size) + file_instance.write_file(auth_token, resource, content, size) @pytest.mark.parametrize( @@ -235,13 +240,14 @@ def test_make_dir(file_instance, status_code, status_message, expected_exception mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "CreateContainer", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.make_dir(resource) + file_instance.make_dir(auth_token, resource) else: - file_instance.make_dir(resource) + file_instance.make_dir(auth_token, resource) @pytest.mark.parametrize( @@ -262,18 +268,19 @@ def test_list_dir( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.infos = infos + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "ListContainer", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - res = file_instance.list_dir(resource) + res = file_instance.list_dir(auth_token, resource) # Lazy evaluation first_item = next(res, None) if first_item is not None: for _ in res: pass else: - res = file_instance.list_dir(resource) + res = file_instance.list_dir(auth_token, resource) # Lazy evaluation first_item = next(res, None) assert first_item == "file1" @@ -303,11 +310,12 @@ def test_read_file( mock_fileget_response = Mock() mock_fileget_response.status_code = 200 mock_fileget_response.iter_content = Mock(return_value=iter_content) + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "InitiateFileDownload", return_value=mock_download_response): if expected_exception: with pytest.raises(expected_exception): - res = file_instance.read_file(resource) + res = file_instance.read_file(auth_token, resource) # Lazy evaluation first_item = next(res, None) if first_item is not None: @@ -315,7 +323,7 @@ def test_read_file( pass else: with patch("requests.get", return_value=mock_fileget_response): - res = file_instance.read_file(resource) + res = file_instance.read_file(auth_token, resource) # Lazy evaluation chunks = list(res) assert chunks == iter_content diff --git a/tests/test_resource.py b/tests/test_resource.py index 4ce89c2..59dd2b8 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -8,13 +8,10 @@ Last updated: 26/07/2024 """ -import sys import unittest import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr -sys.path.append("src/") - -from cs3resource import Resource # noqa: E402 +from cs3client.cs3resource import Resource class TestResource(unittest.TestCase): diff --git a/tests/test_share.py b/tests/test_share.py index 9a80add..295c928 100644 --- a/tests/test_share.py +++ b/tests/test_share.py @@ -5,10 +5,9 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 28/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch import cs3.sharing.collaboration.v1beta1.resources_pb2 as cs3scr @@ -16,25 +15,22 @@ import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.rpc.v1beta1.code_pb2 as cs3code -sys.path.append("src/") -from exceptions.exceptions import ( # noqa: E402 +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, FileLockedException, AlreadyExistsException, ) -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, share_instance, mock_status_code_handler, ) - -# Test cases for the Share class `get_share` method using parameterized tests +# Test cases for the Share class. @pytest.mark.parametrize( @@ -63,16 +59,27 @@ def test_create_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = status_message + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "CreateShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): share_instance.create_share( - resource_info=resource_info, opaque_id=opaque_id, idp=idp, role=role, grantee_type=grantee_type + auth_token, + resource_info=resource_info, + opaque_id=opaque_id, + idp=idp, + role=role, + grantee_type=grantee_type ) else: result = share_instance.create_share( - resource_info=resource_info, opaque_id=opaque_id, idp=idp, role=role, grantee_type=grantee_type + auth_token, + resource_info=resource_info, + opaque_id=opaque_id, + idp=idp, + role=role, + grantee_type=grantee_type ) assert result == expected_result @@ -94,13 +101,14 @@ def test_list_existing_shares( if status_code == cs3code.CODE_OK: mock_response.share_infos = expected_result[0] mock_response.next_page_token = expected_result[1] + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "ListExistingShares", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.list_existing_shares() + share_instance.list_existing_shares(auth_token) else: - result = share_instance.list_existing_shares() + result = share_instance.list_existing_shares(auth_token) assert result == expected_result @@ -123,13 +131,14 @@ def test_get_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "GetShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.get_share(share_id) + share_instance.get_share(auth_token, share_id) else: - result = share_instance.get_share(share_id) + result = share_instance.get_share(auth_token, share_id) assert result == expected_result @@ -152,13 +161,14 @@ def test_remove_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "RemoveShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.remove_share(share_id) + share_instance.remove_share(auth_token, share_id) else: - result = share_instance.remove_share(share_id) + result = share_instance.remove_share(auth_token, share_id) assert result is None @@ -183,13 +193,14 @@ def test_update_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "UpdateShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.update_share(role=role, opaque_id=opaque_id) + share_instance.update_share(auth_token, role=role, opaque_id=opaque_id) else: - result = share_instance.update_share(role=role, opaque_id=opaque_id) + result = share_instance.update_share(auth_token, role=role, opaque_id=opaque_id) assert result == expected_result @@ -211,13 +222,14 @@ def test_list_existing_received_shares( if status_code == cs3code.CODE_OK: mock_response.share_infos = expected_result[0] mock_response.next_page_token = expected_result[1] + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "ListExistingReceivedShares", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.list_received_existing_shares() + share_instance.list_received_existing_shares(auth_token) else: - result = share_instance.list_received_existing_shares() + result = share_instance.list_received_existing_shares(auth_token) assert result == expected_result @@ -240,13 +252,14 @@ def test_get_received_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "GetReceivedShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.get_received_share(share_id) + share_instance.get_received_share(auth_token, share_id) else: - result = share_instance.get_received_share(share_id) + result = share_instance.get_received_share(auth_token, share_id) assert result == expected_result @@ -271,13 +284,14 @@ def test_update_received_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "UpdateReceivedShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.update_received_share(received_share=received_share) + share_instance.update_received_share(auth_token, received_share=received_share) else: - result = share_instance.update_received_share(received_share=received_share) + result = share_instance.update_received_share(auth_token, received_share=received_share) assert result == expected_result @@ -304,13 +318,14 @@ def test_create_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "CreatePublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.create_public_share(resource_info=resource_info, role=role) + share_instance.create_public_share(auth_token, resource_info=resource_info, role=role) else: - result = share_instance.create_public_share(resource_info=resource_info, role=role) + result = share_instance.create_public_share(auth_token, resource_info=resource_info, role=role) assert result == expected_result @@ -331,13 +346,14 @@ def list_existing_public_shares( if status_code == cs3code.CODE_OK: mock_response.share_infos = expected_result[0] mock_response.next_page_token = expected_result[1] + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "ListExistingPublicShares", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.list_existing_public_shares() + share_instance.list_existing_public_shares(auth_token) else: - result = share_instance.list_existing_public_shares() + result = share_instance.list_existing_public_shares(auth_token) assert result == expected_result @@ -360,13 +376,14 @@ def test_get_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "GetPublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.get_public_share(share_id) + share_instance.get_public_share(auth_token, share_id) else: - result = share_instance.get_public_share(share_id) + result = share_instance.get_public_share(auth_token, share_id) assert result == expected_result @@ -391,13 +408,14 @@ def test_update_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "UpdatePublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.update_public_share(type=type, role=role, opaque_id=opqaue_id) + share_instance.update_public_share(auth_token, type=type, role=role, opaque_id=opqaue_id) else: - result = share_instance.update_public_share(type=type, role=role, opaque_id=opqaue_id) + result = share_instance.update_public_share(auth_token, type=type, role=role, opaque_id=opqaue_id) assert result == expected_result @@ -423,13 +441,14 @@ def test_remove_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "RemovePublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.remove_public_share(share_id) + share_instance.remove_public_share(auth_token, share_id) else: - result = share_instance.remove_public_share(share_id) + result = share_instance.remove_public_share(auth_token, share_id) assert result is None diff --git a/tests/test_user.py b/tests/test_user.py index 8c51879..837b183 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -5,32 +5,29 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 28/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch import cs3.rpc.v1beta1.code_pb2 as cs3code -sys.path.append("src/") - -from exceptions.exceptions import ( # noqa: E402 +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, user_instance, mock_status_code_handler, ) - # Test cases for the User class + + @pytest.mark.parametrize( "status_code, status_message, expected_exception, user_data", [ @@ -133,11 +130,12 @@ def test_find_users( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.users = users + auth_token = ('x-access-token', "some_token") with patch.object(user_instance._gateway, "FindUsers", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - user_instance.find_users(filter) + user_instance.find_users(auth_token, filter) else: - result = user_instance.find_users(filter) + result = user_instance.find_users(auth_token, filter) assert result == users