From 7a3126cd980ebc743f4b0d2907acb8810e339e0c Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Fri, 30 Aug 2024 15:18:53 +0200 Subject: [PATCH 1/8] Authentication to be handled externally to allow parallelism --- README.md | 80 ++++++++++++++----------- examples/app_api_example.py | 28 +++++---- examples/auth_example.py | 34 +++++++++++ examples/checkpoints_api_example.py | 25 +++++--- examples/file_api_example.py | 48 ++++++++------- examples/shares_api_example.py | 60 +++++++++++-------- examples/user_api_example.py | 21 +++++-- src/app.py | 21 +++---- src/auth.py | 51 +++++++--------- src/checkpoint.py | 19 +++--- src/cs3client.py | 16 +++-- src/file.py | 65 +++++++++++--------- src/share.py | 92 ++++++++++++++++++----------- src/statuscodehandler.py | 9 +-- src/user.py | 13 ++-- tests/fixtures.py | 79 ++++++++++++------------- tests/test_app.py | 25 ++++---- tests/test_checkpoint.py | 26 ++++---- tests/test_cs3client.py | 22 +------ tests/test_file.py | 65 +++++++++++--------- tests/test_resource.py | 6 +- tests/test_share.py | 88 ++++++++++++++++----------- tests/test_user.py | 19 +++--- 23 files changed, 504 insertions(+), 408 deletions(-) create mode 100644 examples/auth_example.py diff --git a/README.md b/README.md index bdb0292..388a33c 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,7 @@ Alternatively, you can clone this repository and install manually: ```bash git clone git@github.com:cs3org/cs3-python-client.git cd cs3-python-client -pip install -e . -export PYTHONPATH="path/to/cs3-python-client/:$PYTHONPATH" +pip install . ``` @@ -112,73 +111,82 @@ lock_expiration = 1800 To use `cs3client`, you first need to import and configure it. Here's a simple example of how to set up and start using the client. For configuration see [Configuration](#configuration). For more in depth examples see `cs3-python-client/examples/`. -### Initilization +### Initilization and Authentication ```python import logging import configparser from cs3client import CS3Client -from cs3resource import Resource +from auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) - log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") +auth = Auth(client) +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) + ``` ### File Example ```python # mkdir directory_resource = Resource.from_file_ref_and_endpoint(f"/eos/user/r/rwelande/test_directory") -res = client.file.make_dir(directory_resource) +res = client.file.make_dir(auth.get_token(), directory_resource) # touchfile touch_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/touch_file.txt") -res = client.file.touch_file(touch_resource) +res = client.file.touch_file(auth.get_token(), touch_resource) # setxattr resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") -res = client.file.set_xattr(resource, "iop.wopi.lastwritetime", str(1720696124)) +res = client.file.set_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime", str(1720696124)) # rmxattr -res = client.file.remove_xattr(resource, "iop.wopi.lastwritetime") +res = client.file.remove_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime") # stat -res = client.file.stat(resource) +res = client.file.stat(auth.get_token(), resource) # removefile -res = client.file.remove_file(touch_resource) +res = client.file.remove_file(auth.get_token(), touch_resource) # rename rename_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/rename_file.txt") -res = client.file.rename_file(resource, rename_resource) +res = client.file.rename_file(auth.get_token(), resource, rename_resource) # writefile content = b"Hello World" size = len(content) -res = client.file.write_file(rename_resource, content, size) +res = client.file.write_file(auth.get_token(), rename_resource, content, size) # listdir list_directory_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande") -res = client.file.list_dir(list_directory_resource) +res = client.file.list_dir(auth.get_token(), list_directory_resource) # readfile -file_res = client.file.read_file(rename_resource) +file_res = client.file.read_file(auth.get_token(), rename_resource) ``` ### Share Example ```python # Create share # resource = Resource.from_file_ref_and_endpoint("/eos/user/r//text.txt") -resource_info = client.file.stat(resource) +resource_info = client.file.stat(auth.get_token(), resource) user = client.user.get_user_by_claim("username", "") -res = client.share.create_share(resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER") +res = client.share.create_share(auth.get_token(), resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER") # List existing shares # filter_list = [] @@ -186,32 +194,32 @@ filter = client.share.create_share_filter(resource_id=resource_info.id, filter_t filter_list.append(filter) filter = client.share.create_share_filter(share_state="SHARE_STATE_PENDING", filter_type="TYPE_STATE") filter_list.append(filter) -res, _ = client.share.list_existing_shares() +res, _ = client.share.list_existing_shares(auth.get_token(), ) # Get share # share_id = "58" -res = client.share.get_share(opaque_id=share_id) +res = client.share.get_share(auth.get_token(), opaque_id=share_id) # update share # -res = client.share.update_share(opaque_id=share_id, role="VIEWER") +res = client.share.update_share(auth.get_token(), opaque_id=share_id, role="VIEWER") # remove share # -res = client.share.remove_share(opaque_id=share_id) +res = client.share.remove_share(auth.get_token(), opaque_id=share_id) # List existing received shares # filter_list = [] filter = client.share.create_share_filter(share_state="SHARE_STATE_ACCEPTED", filter_type="TYPE_STATE") filter_list.append(filter) -res, _ = client.share.list_received_existing_shares() +res, _ = client.share.list_received_existing_shares(auth.get_token()) # get received share # -received_share = client.share.get_received_share(opaque_id=share_id) +received_share = client.share.get_received_share(auth.get_token(), opaque_id=share_id) # update recieved share # -res = client.share.update_received_share(received_share=received_share, state="SHARE_STATE_ACCEPTED") +res = client.share.update_received_share(auth.get_token(), received_share=received_share, state="SHARE_STATE_ACCEPTED") # create public share # -res = client.share.create_public_share(resource_info, role="VIEWER") +res = client.share.create_public_share(auth.get_token(), resource_info, role="VIEWER") # list existing public shares # filter_list = [] @@ -219,22 +227,22 @@ filter = client.share.create_public_share_filter(resource_id=resource_info.id, f filter_list.append(filter) res, _ = client.share.list_existing_public_shares(filter_list=filter_list) -res = client.share.get_public_share(opaque_id=share_id, sign=True) +res = client.share.get_public_share(auth.get_token(), opaque_id=share_id, sign=True) # OR token = "" # res = client.share.get_public_share(token=token, sign=True) # update public share # -res = client.share.update_public_share(type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello") +res = client.share.update_public_share(auth.get_token(), type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello") # remove public share # -res = client.share.remove_public_share(token=token) +res = client.share.remove_public_share(auth.get_token(), token=token) ``` ### User Example ```python # find_user -res = client.user.find_users("rwel") +res = client.user.find_users(auth.get_token(), "rwel") # get_user res = client.user.get_user("https://auth.cern.ch/auth/realms/cern", "asdoiqwe") @@ -253,21 +261,21 @@ res = client.user.get_user_by_claim("username", "rwelande") ### App Example ```python # list_app_providers -res = client.app.list_app_providers() +res = client.app.list_app_providers(auth.get_token()) # open_in_app resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/collabora.odt") -res = client.app.open_in_app(resource) +res = client.app.open_in_app(auth.get_token(), resource) ``` ### Checkpoint Example ```python # list file versions resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/test.md") -res = client.checkpoint.list_file_versions(resource) +res = client.checkpoint.list_file_versions(auth.get_token(), resource) # restore file version -res = client.checkpoint.restore_file_version(resource, "1722936250.0569fa2f") +res = client.checkpoint.restore_file_version(auth.get_token(), resource, "1722936250.0569fa2f") ``` ## Documentation diff --git a/examples/app_api_example.py b/examples/app_api_example.py index 202754e..ca7bf1f 100644 --- a/examples/app_api_example.py +++ b/examples/app_api_example.py @@ -6,35 +6,41 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging import configparser -from cs3client import CS3Client from cs3resource import Resource +from cs3client import CS3Client +from auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) - client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") - -print(client.auth.get_token()) +auth = Auth(client) +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) # list_app_providers -res = client.app.list_app_providers() +res = client.app.list_app_providers(auth.get_token()) if res is not None: print(res) # open_in_app resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/collabora.odt") -res = client.app.open_in_app(resource) +res = client.app.open_in_app(auth.get_token(), resource) if res is not None: print(res) diff --git a/examples/auth_example.py b/examples/auth_example.py new file mode 100644 index 0000000..2763242 --- /dev/null +++ b/examples/auth_example.py @@ -0,0 +1,34 @@ +""" +auth_example.py + +Example script to demonstrate the usage of the app API in the CS3Client class. +note that these are examples, and is not meant to be run as a script. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 30/08/2024 +""" + +import logging +import configparser +from cs3client import CS3Client +from auth import Auth + +config = configparser.ConfigParser() +with open("default.conf") as fdef: + config.read_file(fdef) +log = logging.getLogger(__name__) + +client = CS3Client(config, "cs3client", log) +auth = Auth(client) +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) diff --git a/examples/checkpoints_api_example.py b/examples/checkpoints_api_example.py index db4c094..e164135 100644 --- a/examples/checkpoints_api_example.py +++ b/examples/checkpoints_api_example.py @@ -6,36 +6,45 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging import configparser -from cs3client import CS3Client from cs3resource import Resource +from cs3client import CS3Client +from auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") +auth = Auth(client) +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None markdown_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/test.md") -res = client.checkpoint.list_file_versions(markdown_resource) +res = client.checkpoint.list_file_versions(auth.get_token(), markdown_resource) if res is not None: for ver in res: print(ver) -res = client.checkpoint.restore_file_version(markdown_resource, "1722936250.0569fa2f") +res = client.checkpoint.restore_file_version(auth.get_token(), markdown_resource, "1722936250.0569fa2f") if res is not None: for ver in res: print(ver) diff --git a/examples/file_api_example.py b/examples/file_api_example.py index 468449b..74aec07 100644 --- a/examples/file_api_example.py +++ b/examples/file_api_example.py @@ -13,76 +13,82 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 01/08/2024 +Last updated: 30/08/2024 """ import logging import configparser from cs3client import CS3Client from cs3resource import Resource +from auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -client.auth.set_token("") -# OR -# client.auth.set_client_secret("") - -# Authentication -print(client.auth.get_token()) +auth = Auth(client) +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None # mkdir for i in range(1, 4): directory_resource = Resource.from_file_ref_and_endpoint(f"/eos/user/r/rwelande/test_directory{i}") - res = client.file.make_dir(directory_resource) + res = client.file.make_dir(auth.get_token(), directory_resource) if res is not None: print(res) # touchfile touch_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/touch_file.txt") text_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") -res = client.file.touch_file(touch_resource) -res = client.file.touch_file(text_resource) +res = client.file.touch_file(auth.get_token(), touch_resource) +res = client.file.touch_file(auth.get_token(), text_resource) if res is not None: print(res) # setxattr resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") -res = client.file.set_xattr(resource, "iop.wopi.lastwritetime", str(1720696124)) +res = client.file.set_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime", str(1720696124)) if res is not None: print(res) # rmxattr -res = client.file.remove_xattr(resource, "iop.wopi.lastwritetime") +res = client.file.remove_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime") if res is not None: print(res) # stat -res = client.file.stat(text_resource) +res = client.file.stat(auth.get_token(), text_resource) if res is not None: print(res) # removefile -res = client.file.remove_file(touch_resource) +res = client.file.remove_file(auth.get_token(), touch_resource) if res is not None: print(res) -res = client.file.touch_file(touch_resource) +res = client.file.touch_file(auth.get_token(), touch_resource) # rename rename_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/rename_file.txt") -res = client.file.rename_file(resource, rename_resource) +res = client.file.rename_file(auth.get_token(), resource, rename_resource) if res is not None: print(res) @@ -90,20 +96,20 @@ # writefile content = b"Hello World" size = len(content) -res = client.file.write_file(rename_resource, content, size) +res = client.file.write_file(auth.get_token(), rename_resource, content, size) if res is not None: print(res) # rmdir (same as deletefile) -res = client.file.remove_file(directory_resource) +res = client.file.remove_file(auth.get_token(), directory_resource) if res is not None: print(res) # listdir list_directory_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande") -res = client.file.list_dir(list_directory_resource) +res = client.file.list_dir(auth.get_token(), list_directory_resource) first_item = next(res, None) if first_item is not None: @@ -114,7 +120,7 @@ print("empty response") # readfile -file_res = client.file.read_file(rename_resource) +file_res = client.file.read_file(auth.get_token(), rename_resource) content = b"" try: for chunk in file_res: diff --git a/examples/shares_api_example.py b/examples/shares_api_example.py index 5a42c8d..03d6dbf 100644 --- a/examples/shares_api_example.py +++ b/examples/shares_api_example.py @@ -6,43 +6,53 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging import configparser from cs3client import CS3Client from cs3resource import Resource +from auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") - -# Authentication -print(client.auth.get_token()) +auth = Auth(client) +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None # Create share # resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text.txt") -resource_info = client.file.stat(resource) +resource_info = client.file.stat(auth.get_token(), resource) # VIEWER user = client.user.get_user_by_claim("mail", "diogo.castro@cern.ch") -res = client.share.create_share(resource_info, user.id.opaque_id, user.id.idp, "VIEWER", "USER") +res = client.share.create_share( + auth.get_token(), resource_info, user.id.opaque_id, user.id.idp, "VIEWER", "USER" +) if res is not None: print(res) # EDITOR user = client.user.get_user_by_claim("username", "lopresti") -res = client.share.create_share(resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER") +res = client.share.create_share( + auth.get_token(), resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER" +) if res is not None: print(res) @@ -54,26 +64,26 @@ filter_list.append(filter) filter = client.share.create_share_filter(share_state="SHARE_STATE_PENDING", filter_type="TYPE_STATE") filter_list.append(filter) -res, _ = client.share.list_existing_shares() +res, _ = client.share.list_existing_shares(auth.get_token(), filter_list=filter_list) if res is not None: for share_info in res: print(share_info.share) # Get share # share_id = "58" -res = client.share.get_share(opaque_id=share_id) +res = client.share.get_share(auth.get_token(), opaque_id=share_id) if res is not None: print(res) # update share # share_id = "58" -res = client.share.update_share(opaque_id=share_id, role="VIEWER") +res = client.share.update_share(auth.get_token(), opaque_id=share_id, role="VIEWER") if res is not None: print(res) # remove share # share_id = "58" -res = client.share.remove_share(opaque_id=share_id) +res = client.share.remove_share(auth.get_token(), opaque_id=share_id) if res is not None: print(res) @@ -87,7 +97,7 @@ filter_list.append(filter) # NOTE: filters for received shares are not implemented (14/08/2024), therefore it is left out -res, _ = client.share.list_received_existing_shares() +res, _ = client.share.list_received_existing_shares(auth.get_token()) if res is not None: for share_info in res: print(share_info.received_share) @@ -95,17 +105,19 @@ # get received share # share_id = "43" -received_share = client.share.get_received_share(opaque_id=share_id) +received_share = client.share.get_received_share(auth.get_token(), opaque_id=share_id) if received_share is not None: print(received_share) # update recieved share # -res = client.share.update_received_share(received_share=received_share, state="SHARE_STATE_ACCEPTED") +res = client.share.update_received_share( + auth.get_token(), received_share=received_share, state="SHARE_STATE_ACCEPTED" +) if res is not None: print(res) # create public share # -res = client.share.create_public_share(resource_info, role="VIEWER") +res = client.share.create_public_share(auth.get_token(), resource_info, role="VIEWER") if res is not None: print(res) @@ -116,7 +128,7 @@ filter = client.share.create_public_share_filter(resource_id=resource_info.id, filter_type="TYPE_RESOURCE_ID") filter_list.append(filter) print(filter_list) -res, _ = client.share.list_existing_public_shares(filter_list=filter_list) +res, _ = client.share.list_existing_public_shares(auth.get_token(), filter_list=filter_list) if res is not None: for share_info in res: print(share_info.share) @@ -125,16 +137,18 @@ share_id = "63" # OR token = "7FbP1EBXJQTqK0d" -res = client.share.get_public_share(opaque_id=share_id, sign=True) +res = client.share.get_public_share(auth.get_token(), opaque_id=share_id, sign=True) if res is not None: print(res) # update public share # -res = client.share.update_public_share(type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello") +res = client.share.update_public_share( + auth.get_token(), type="TYPE_PASSWORD", token=token, role="VIEWER", password="hello" +) if res is not None: print(res) # remove public share # -res = client.share.remove_public_share(token=token) +res = client.share.remove_public_share(auth.get_token(), token=token) if res is not None: print(res) diff --git a/examples/user_api_example.py b/examples/user_api_example.py index c4a6973..d368df5 100644 --- a/examples/user_api_example.py +++ b/examples/user_api_example.py @@ -6,29 +6,38 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 02/08/2024 +Last updated: 30/08/2024 """ import logging import configparser from cs3client import CS3Client +from auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: config.read_file(fdef) -# log log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) -# client.auth.set_token("") -# OR -client.auth.set_client_secret("") +auth = Auth(client) +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) res = None # find_user -res = client.user.find_users("rwel") +res = client.user.find_users(client.auth.get_token(), "rwel") if res is not None: print(res) diff --git a/src/app.py b/src/app.py index d54ce22..d10e418 100644 --- a/src/app.py +++ b/src/app.py @@ -3,17 +3,17 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging -from auth import Auth -from cs3resource import Resource import cs3.app.registry.v1beta1.registry_api_pb2 as cs3arreg import cs3.app.registry.v1beta1.resources_pb2 as cs3arres import cs3.gateway.v1beta1.gateway_api_pb2 as cs3gw import cs3.app.provider.v1beta1.resources_pb2 as cs3apr from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub + +from cs3resource import Resource from statuscodehandler import StatusCodeHandler from config import Config @@ -28,7 +28,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -37,20 +36,21 @@ def __init__( :param config: Config object containing the configuration parameters. :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. :param status_code_handler: An instance of the StatusCodeHandler class. """ self._status_code_handler: StatusCodeHandler = status_code_handler self._gateway: GatewayAPIStub = gateway self._log: logging.Logger = log self._config: Config = config - self._auth: Auth = auth - def open_in_app(self, resource: Resource, view_mode: str = None, app: str = None) -> cs3apr.OpenInAppURL: + def open_in_app( + self, auth_token: tuple, resource: Resource, view_mode: str = None, app: str = None + ) -> cs3apr.OpenInAppURL: """ Open a file in an app, given the resource, view mode (VIEW_MODE_VIEW_ONLY, VIEW_MODE_READ_ONLY, VIEW_MODE_READ_WRITE, VIEW_MODE_PREVIEW), and app name. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource object containing the resource information. :param view_mode: View mode of the app. :param app: App name. @@ -63,21 +63,22 @@ def open_in_app(self, resource: Resource, view_mode: str = None, app: str = None if view_mode: view_mode_type = cs3gw.OpenInAppRequest.ViewMode.Value(view_mode) req = cs3gw.OpenInAppRequest(ref=resource.ref, view_mode=view_mode_type, app=app) - res = self._gateway.OpenInApp(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.OpenInApp(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "open in app", f"{resource.get_file_ref_str()}") self._log.debug(f'msg="Invoked OpenInApp" {resource.get_file_ref_str()} trace="{res.status.trace}"') return res.OpenInAppURL - def list_app_providers(self) -> list[cs3arres.ProviderInfo]: + def list_app_providers(self, auth_token: dict) -> list[cs3arres.ProviderInfo]: """ list_app_providers lists all the app providers. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :return: List of app providers. :raises: AuthenticationException (Operation not permitted) :raises: UnknownException (Unknown error) """ req = cs3arreg.ListAppProvidersRequest() - res = self._gateway.ListAppProviders(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListAppProviders(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list app providers") self._log.debug(f'msg="Invoked ListAppProviders" res_count="{len(res.providers)}" trace="{res.status.trace}"') return res.providers diff --git a/src/auth.py b/src/auth.py index 94b0714..b8da76d 100644 --- a/src/auth.py +++ b/src/auth.py @@ -3,7 +3,7 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import grpc @@ -15,6 +15,7 @@ from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub from cs3.rpc.v1beta1.code_pb2 import CODE_OK +from cs3client import CS3Client from exceptions.exceptions import AuthenticationException, SecretNotSetException from config import Config @@ -24,7 +25,7 @@ class Auth: Auth class to handle authentication and token validation with CS3 Gateway API. """ - def __init__(self, config: Config, log: logging.Logger, gateway: GatewayAPIStub) -> None: + def __init__(self, cs3_client: CS3Client) -> None: """ Initializes the Auth class with configuration, logger, and gateway stub, NOTE that token OR the client secret has to be set when instantiating the auth object. @@ -33,23 +34,13 @@ def __init__(self, config: Config, log: logging.Logger, gateway: GatewayAPIStub) :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. """ - self._gateway: GatewayAPIStub = gateway - self._log: logging.Logger = log - self._config: Config = config + self._gateway: GatewayAPIStub = cs3_client._gateway + self._log: logging.Logger = cs3_client._log + self._config: Config = cs3_client._config # The user should be able to change the client secret (e.g. token) at runtime self._client_secret: str | None = None self._token: str | None = None - def set_token(self, token: str) -> None: - """ - Should be used if the user wishes to set the reva token directly, instead of letting the client - exchange credentials for the token. NOTE that token OR the client secret has to be set when - instantiating the client object. - - :param token: The reva token. - """ - self._token = token - def set_client_secret(self, token: str) -> None: """ Sets the client secret, exists so that the user can change the client secret (e.g. token, password) at runtime, @@ -71,16 +62,14 @@ def get_token(self) -> tuple[str, str]: :raises: SecretNotSetException (neither token or client secret was set) """ - if not Auth._check_token(self._token): - # Check that client secret or token is set - if not self._client_secret and not self._token: - self._log.error("Attempted to authenticate, neither client secret or token was set.") - raise SecretNotSetException("The client secret (e.g. token, passowrd) is not set") - elif not self._client_secret and self._token: - # Case where ONLY a token is provided but it has expired - self._log.error("The provided token have expired") - raise AuthenticationException("The credentials have expired") - # Create an authentication request + if not self._client_secret: + self._log.error("Attempted to authenticate, client secret was not set") + raise SecretNotSetException("The client secret (e.g. token, passowrd) is not set") + + try: + self.check_token(self._token) + except AuthenticationException: + # Token has expired or has not been set, obtain another one. req = AuthenticateRequest( type=self._config.auth_login_type, client_id=self._config.auth_client_id, @@ -116,20 +105,22 @@ def list_auth_providers(self) -> list[str]: return res.types @classmethod - def _check_token(cls, token: str) -> bool: + def check_token(cls, token: str) -> tuple: """ Checks if the given token is set and valid. :param token: JWT token as a string. - :return: True if the token is valid, False otherwise. + :return tuple: A tuple containing the header key and the token. + :raises: ValueError (Token missing) + :raises: AuthenticationException (Token is expired) """ if not token: - return False + raise AuthenticationException("Token not set") # Decode the token without verifying the signature decoded_token = jwt.decode(jwt=token, algorithms=["HS256"], options={"verify_signature": False}) now = datetime.datetime.now().timestamp() token_expiration = decoded_token.get("exp") if token_expiration and now > token_expiration: - return False + raise AuthenticationException("Token has expired") - return True + return ("x-access-token", token) diff --git a/src/checkpoint.py b/src/checkpoint.py index 9e4cbdd..ea9bcca 100644 --- a/src/checkpoint.py +++ b/src/checkpoint.py @@ -3,15 +3,15 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ from typing import Generator import logging -from auth import Auth import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3spp from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub + from config import Config from statuscodehandler import StatusCodeHandler from cs3resource import Resource @@ -27,7 +27,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -36,21 +35,20 @@ def __init__( :param config: Config object containing the configuration parameters. :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. :param status_code_handler: An instance of the StatusCodeHandler class. """ self._gateway: GatewayAPIStub = gateway self._log: logging.Logger = log self._config: Config = config - self._auth: Auth = auth self._status_code_handler: StatusCodeHandler = status_code_handler def list_file_versions( - self, resource: Resource, page_token: str = "", page_size: int = 0 + self, auth_token: tuple, resource: Resource, page_token: str = "", page_size: int = 0 ) -> Generator[cs3spr.FileVersion, any, any]: """ List all versions of a file. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource object containing the resource information. :param page_token: Token for pagination. :param page_size: Number of file versions to return. @@ -61,15 +59,18 @@ def list_file_versions( :raises: UnknownException (Unknown error) """ req = cs3spp.ListFileVersionsRequest(ref=resource.ref, page_token=page_token, page_size=page_size) - res = self._gateway.ListFileVersions(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListFileVersions(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list file versions", f"{resource.get_file_ref_str()}") self._log.debug(f'msg="list file versions" {resource.get_file_ref_str()} trace="{res.status.trace}"') return res.versions - def restore_file_version(self, resource: Resource, version_key: str, lock_id: str = None) -> None: + def restore_file_version( + self, auth_token: tuple, resource: Resource, version_key: str, lock_id: str = None + ) -> None: """ Restore a file to a previous version. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource object containing the resource information. :param version_key: Key of the version to restore. :param lock_id: Lock ID of the file (OPTIONAL). @@ -79,7 +80,7 @@ def restore_file_version(self, resource: Resource, version_key: str, lock_id: st :raises: UnknownException (Unknown error) """ req = cs3spp.RestoreFileVersionRequest(ref=resource.ref, key=version_key, lock_id=lock_id) - res = self._gateway.RestoreFileVersion(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.RestoreFileVersion(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "restore file version", f"{resource.get_file_ref_str()}") self._log.debug(f'msg="restore file version" {resource.get_file_ref_str()} trace="{res.status.trace}"') return diff --git a/src/cs3client.py b/src/cs3client.py index de2e523..cfaf301 100644 --- a/src/cs3client.py +++ b/src/cs3client.py @@ -3,15 +3,14 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import grpc import logging import cs3.gateway.v1beta1.gateway_api_pb2_grpc as cs3gw_grpc - from configparser import ConfigParser -from auth import Auth + from file import File from user import User from share import Share @@ -46,14 +45,13 @@ def __init__(self, config: ConfigParser, config_category: str, log: logging.Logg self._gateway: cs3gw_grpc.GatewayAPIStub = cs3gw_grpc.GatewayAPIStub(self.channel) self._status_code_handler: StatusCodeHandler = StatusCodeHandler(self._log, self._config) - self.auth: Auth = Auth(self._config, self._log, self._gateway) - self.file: File = File(self._config, self._log, self._gateway, self.auth, self._status_code_handler) - self.user: User = User(self._config, self._log, self._gateway, self.auth, self._status_code_handler) - self.app: App = App(self._config, self._log, self._gateway, self.auth, self._status_code_handler) + self.file: File = File(self._config, self._log, self._gateway, self._status_code_handler) + self.user: User = User(self._config, self._log, self._gateway, self._status_code_handler) + self.app: App = App(self._config, self._log, self._gateway, self._status_code_handler) self.checkpoint: Checkpoint = Checkpoint( - self._config, self._log, self._gateway, self.auth, self._status_code_handler + self._config, self._log, self._gateway, self._status_code_handler ) - self.share = Share(self._config, self._log, self._gateway, self.auth, self._status_code_handler) + self.share = Share(self._config, self._log, self._gateway, self._status_code_handler) def _create_channel(self) -> grpc.Channel: """ diff --git a/src/file.py b/src/file.py index 8abd9b6..2201f57 100644 --- a/src/file.py +++ b/src/file.py @@ -3,23 +3,22 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 29/08/2024 """ import time import logging import http import requests +from typing import Generator import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3sp from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub import cs3.types.v1beta1.types_pb2 as types from config import Config -from typing import Generator from exceptions.exceptions import AuthenticationException, FileLockedException from cs3resource import Resource -from auth import Auth from statuscodehandler import StatusCodeHandler @@ -29,7 +28,7 @@ class File: """ def __init__( - self, config: Config, log: logging.Logger, gateway: GatewayAPIStub, auth: Auth, + self, config: Config, log: logging.Logger, gateway: GatewayAPIStub, status_code_handler: StatusCodeHandler ) -> None: """ @@ -38,19 +37,18 @@ def __init__( :param config: Config object containing the configuration parameters. :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. :param status_code_handler: An instance of the StatusCodeHandler class. """ - self._auth: Auth = auth self._config: Config = config self._log: logging.Logger = log self._gateway: GatewayAPIStub = gateway self._status_code_handler: StatusCodeHandler = status_code_handler - def stat(self, resource: Resource) -> cs3spr.ResourceInfo: + def stat(self, auth_token: tuple, resource: Resource) -> cs3spr.ResourceInfo: """ Stat a file and return the ResourceInfo object. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: resource to stat. :return: cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo (success) :raises: NotFoundException (File not found) @@ -59,7 +57,7 @@ def stat(self, resource: Resource) -> cs3spr.ResourceInfo: """ tstart = time.time() - res = self._gateway.Stat(request=cs3sp.StatRequest(ref=resource.ref), metadata=[self._auth.get_token()]) + res = self._gateway.Stat(request=cs3sp.StatRequest(ref=resource.ref), metadata=[auth_token]) tend = time.time() self._status_code_handler.handle_errors(res.status, "stat", resource.get_file_ref_str()) self._log.info( @@ -68,10 +66,11 @@ def stat(self, resource: Resource) -> cs3spr.ResourceInfo: ) return res.info - def set_xattr(self, resource: Resource, key: str, value: str) -> None: + def set_xattr(self, auth_token: tuple, resource: Resource, key: str, value: str) -> None: """ Set the extended attribute to for a resource. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: resource that has the attribute. :param key: attribute key. :param value: value to set. @@ -83,16 +82,17 @@ def set_xattr(self, resource: Resource, key: str, value: str) -> None: md = cs3spr.ArbitraryMetadata() md.metadata.update({key: value}) # pylint: disable=no-member req = cs3sp.SetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata=md) - res = self._gateway.SetArbitraryMetadata(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.SetArbitraryMetadata(request=req, metadata=[auth_token]) # CS3 storages may refuse to set an xattr in case of lock mismatch: this is an overprotection, # as the lock should concern the file's content, not its metadata, however we need to handle that self._status_code_handler.handle_errors(res.status, "set extended attribute", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked setxattr" trace="{res.status.trace}"') - def remove_xattr(self, resource: Resource, key: str) -> None: + def remove_xattr(self, auth_token: tuple, resource: Resource, key: str) -> None: """ Remove the extended attribute . + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: cs3client resource. :param key: key for attribute to remove. :return: None (Success) @@ -101,14 +101,15 @@ def remove_xattr(self, resource: Resource, key: str) -> None: :raises: UnknownException (Unknown error) """ req = cs3sp.UnsetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata_keys=[key]) - res = self._gateway.UnsetArbitraryMetadata(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.UnsetArbitraryMetadata(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "remove extended attribute", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked UnsetArbitraryMetaData" trace="{res.status.trace}"') - def rename_file(self, resource: Resource, newresource: Resource) -> None: + def rename_file(self, auth_token: tuple, resource: Resource, newresource: Resource) -> None: """ Rename/move resource to new resource. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Original resource. :param newresource: New resource. :return: None (Success) @@ -118,14 +119,15 @@ def rename_file(self, resource: Resource, newresource: Resource) -> None: :raises: UnknownException (Unknown Error) """ req = cs3sp.MoveRequest(source=resource.ref, destination=newresource.ref) - res = self._gateway.Move(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.Move(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "rename file", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked Move" trace="{res.status.trace}"') - def remove_file(self, resource: Resource) -> None: + def remove_file(self, auth_token: tuple, resource: Resource) -> None: """ Remove a resource. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource to remove. :return: None (Success) :raises: AuthenticationException (Authentication Failed) @@ -133,14 +135,15 @@ def remove_file(self, resource: Resource) -> None: :raises: UnknownException (Unknown error) """ req = cs3sp.DeleteRequest(ref=resource.ref) - res = self._gateway.Delete(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.Delete(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "remove file", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked Delete" trace="{res.status.trace}"') - def touch_file(self, resource: Resource) -> None: + def touch_file(self, auth_token: tuple, resource: Resource) -> None: """ Create a resource. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource to create. :return: None (Success) :raises: FileLockedException (File is locked) @@ -151,17 +154,18 @@ def touch_file(self, resource: Resource) -> None: ref=resource.ref, opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode("0"))}), ) - res = self._gateway.TouchFile(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.TouchFile(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "touch file", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked TouchFile" trace="{res.status.trace}"') - def write_file(self, resource: Resource, content: str | bytes, size: int) -> None: + def write_file(self, auth_token: tuple, resource: Resource, content: str | bytes, size: int) -> None: """ Write a file using the given userid as access token. The entire content is written and any pre-existing file is deleted (or moved to the previous version if supported), writing a file with size 0 is equivalent to "touch file" and should be used if the implementation does not support touchfile. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource to write content to. :param content: content to write :param size: size of content (optional) @@ -181,7 +185,7 @@ def write_file(self, resource: Resource, content: str | bytes, size: int) -> Non ref=resource.ref, opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode(str(size)))}), ) - res = self._gateway.InitiateFileUpload(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.InitiateFileUpload(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "write file", resource.get_file_ref_str()) tend = time.time() self._log.debug( @@ -197,13 +201,13 @@ def write_file(self, resource: Resource, content: str | bytes, size: int) -> Non "File-Path": resource.file, "File-Size": str(size), "X-Reva-Transfer": protocol.token, - **dict([self._auth.get_token()]), + **dict([auth_token]), } else: headers = { "Upload-Length": str(size), "X-Reva-Transfer": protocol.token, - **dict([self._auth.get_token()]), + **dict([auth_token]), } putres = requests.put( url=protocol.upload_endpoint, @@ -245,10 +249,11 @@ def write_file(self, resource: Resource, content: str | bytes, size: int) -> Non f'elapsedTimems="{(tend - tstart) * 1000:.1f}"' ) - def read_file(self, resource: Resource) -> Generator[bytes, None, None]: + def read_file(self, auth_token: tuple, resource: Resource) -> Generator[bytes, None, None]: """ Read a file. Note that the function is a generator, managed by the app server. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Resource to read. :return: Generator[Bytes, None, None] (Success) :raises: NotFoundException (Resource not found) @@ -259,7 +264,7 @@ def read_file(self, resource: Resource) -> Generator[bytes, None, None]: # prepare endpoint req = cs3sp.InitiateFileDownloadRequest(ref=resource.ref) - res = self._gateway.InitiateFileDownload(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.InitiateFileDownload(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "read file", resource.get_file_ref_str()) tend = time.time() self._log.debug( @@ -269,7 +274,7 @@ def read_file(self, resource: Resource) -> Generator[bytes, None, None]: # Download try: protocol = [p for p in res.protocols if p.protocol in ["simple", "spaces"]][0] - headers = {"X-Reva-Transfer": protocol.token, **dict([self._auth.get_token()])} + headers = {"X-Reva-Transfer": protocol.token, **dict([auth_token])} fileget = requests.get( url=protocol.download_endpoint, headers=headers, @@ -294,10 +299,11 @@ def read_file(self, resource: Resource) -> Generator[bytes, None, None]: for chunk in data: yield chunk - def make_dir(self, resource: Resource) -> None: + def make_dir(self, auth_token: tuple, resource: Resource) -> None: """ Create a directory. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: Direcotry to create. :return: None (Success) :raises: FileLockedException (File is locked) @@ -305,16 +311,17 @@ def make_dir(self, resource: Resource) -> None: :raises: UnknownException (Unknown error) """ req = cs3sp.CreateContainerRequest(ref=resource.ref) - res = self._gateway.CreateContainer(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.CreateContainer(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "make directory", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked CreateContainer" trace="{res.status.trace}"') def list_dir( - self, resource: Resource + self, auth_token: tuple, resource: Resource ) -> Generator[cs3spr.ResourceInfo, None, None]: """ List the contents of a directory, note that the function is a generator. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: the directory. :return: Generator[cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo, None, None] (Success) :raises: NotFoundException (Resrouce not found) @@ -322,7 +329,7 @@ def list_dir( :raises: UnknownException (Unknown error) """ req = cs3sp.ListContainerRequest(ref=resource.ref) - res = self._gateway.ListContainer(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListContainer(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list directory", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked ListContainer" trace="{res.status.trace}"') for info in res.infos: diff --git a/src/share.py b/src/share.py index 81f1254..cc86819 100644 --- a/src/share.py +++ b/src/share.py @@ -3,16 +3,10 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging -from auth import Auth -from cs3resource import Resource -from config import Config -from statuscodehandler import StatusCodeHandler - - import cs3.sharing.collaboration.v1beta1.collaboration_api_pb2 as cs3scapi from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub import cs3.sharing.collaboration.v1beta1.resources_pb2 as cs3scr @@ -24,6 +18,10 @@ import google.protobuf.field_mask_pb2 as field_masks import cs3.types.v1beta1.types_pb2 as cs3types +from cs3resource import Resource +from config import Config +from statuscodehandler import StatusCodeHandler + class Share: """ @@ -35,7 +33,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -44,21 +41,26 @@ def __init__( :param config: Config object containing the configuration parameters. :param log: Logger instance for logging. :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param auth: An instance of the auth class. :param status_code_handler: An instance of the StatusCodeHandler class. """ self._status_code_handler: StatusCodeHandler = status_code_handler self._gateway: GatewayAPIStub = gateway self._log: logging.Logger = log self._config: Config = config - self._auth: Auth = auth def create_share( - self, resource_info: cs3spr.ResourceInfo, opaque_id: str, idp: str, role: str, grantee_type: str + self, + auth_token: tuple, + resource_info: cs3spr.ResourceInfo, + opaque_id: str, + idp: str, + role: str, + grantee_type: str ) -> cs3scr.Share: """ Create a share for a resource to the user/group with the specified role, using their opaque id and idp. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource_info: Resource info, see file.stat (REQUIRED). :param opaque_id: Opaque group/user id, (REQUIRED). :param idp: Identity provider, (REQUIRED). @@ -74,7 +76,7 @@ def create_share( """ share_grant = Share._create_share_grant(opaque_id, idp, role, grantee_type) req = cs3scapi.CreateShareRequest(resource_info=resource_info, grant=share_grant) - res = self._gateway.CreateShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.CreateShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "create share", f'opaque_id="{opaque_id}" resource_id="{resource_info.id}"' ) @@ -85,11 +87,12 @@ def create_share( return res.share def list_existing_shares( - self, filter_list: list[cs3scr.Filter] = None, page_size: int = 0, page_token: str = None + self, auth_token: tuple, filter_list: list[cs3scr.Filter] = None, page_size: int = 0, page_token: str = None ) -> list[cs3scr.Share]: """ List shares based on a filter. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter object to filter the shares, see create_share_filter. :param page_size: Number of shares to return in a page, defaults to 0, server decides. :param page_token: Token to get to a specific page. @@ -98,7 +101,7 @@ def list_existing_shares( :raises: UnknownException (Unknown error) """ req = cs3scapi.ListSharesRequest(filters=filter_list, page_size=page_size, page_token=page_token) - res = self._gateway.ListExistingShares(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListExistingShares(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list existing shares", f'filter="{filter_list}"') self._log.debug( f'msg="Invoked ListExistingShares" filter="{filter_list}" res_count="{len(res.share_infos)}' @@ -106,11 +109,12 @@ def list_existing_shares( ) return (res.share_infos, res.next_page_token) - def get_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> cs3scr.Share: + def get_share(self, auth_token: tuple, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> cs3scr.Share: """ Get a share by its opaque id or share key (combination of resource_id, grantee and owner), one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :return: Share object. @@ -127,7 +131,7 @@ def get_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> else: raise ValueError("opaque_id or share_key is required") - res = self._gateway.GetShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.GetShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "get share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -137,11 +141,12 @@ def get_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> ) return res.share - def remove_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> None: + def remove_share(self, auth_token: tuple, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> None: """ Remove a share by its opaque id or share key (combination of resource_id, grantee and owner), one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :return: None @@ -157,7 +162,7 @@ def remove_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) req = cs3scapi.RemoveShareRequest(ref=cs3scr.ShareReference(key=share_key)) else: raise ValueError("opaque_id or share_key is required") - res = self._gateway.RemoveShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.RemoveShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "remove share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -168,11 +173,16 @@ def remove_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) return def update_share( - self, role: str, opaque_id: str = None, share_key: cs3scr.ShareKey = None, display_name: str = None + self, auth_token: tuple, + role: str, + opaque_id: str = None, + share_key: cs3scr.ShareKey = None, + display_name: str = None ) -> cs3scr.Share: """ Update a share by its opaque id. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id. (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :param role: Role to update the share, VIEWER or EDITOR (REQUIRED). @@ -195,7 +205,7 @@ def update_share( raise ValueError("opaque_id or share_key is required") req = cs3scapi.UpdateShareRequest(ref=ref, field=update) - res = self._gateway.UpdateShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.UpdateShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "update share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -206,12 +216,13 @@ def update_share( return res.share def list_received_existing_shares( - self, filter_list: list = None, page_size: int = 0, page_token: str = None + self, auth_token: tuple, filter_list: list = None, page_size: int = 0, page_token: str = None ) -> list: """ List received existing shares. NOTE: Filters for received shares are not yet implemented (14/08/2024) + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter object to filter the shares, see create_share_filter. :param page_size: Number of shares to return in a page, defaults to 0, server decides. :param page_token: Token to get to a specific page. @@ -220,7 +231,7 @@ def list_received_existing_shares( :raises: UnknownException (Unknown error) """ req = cs3scapi.ListReceivedSharesRequest(filters=filter_list, page_size=page_size, page_token=page_token) - res = self._gateway.ListExistingReceivedShares(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListExistingReceivedShares(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list received existing shares", f'filter="{filter_list}"') self._log.debug( f'msg="Invoked ListExistingReceivedShares" filter="{filter_list}" res_count="{len(res.share_infos)}"' @@ -228,11 +239,14 @@ def list_received_existing_shares( ) return (res.share_infos, res.next_page_token) - def get_received_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = None) -> cs3scr.ReceivedShare: + def get_received_share( + self, auth_token: tuple, opaque_id: str = None, share_key: cs3scr.ShareKey = None + ) -> cs3scr.ReceivedShare: """ Get a received share by its opaque id or share key (combination of resource_id, grantee and owner), one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id. (SEMI-OPTIONAL). :param share_key: Share key, see ShareKey definition in cs3apis (SEMI-OPTIONAL). :return: ReceivedShare object. @@ -248,7 +262,7 @@ def get_received_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = req = cs3scapi.GetReceivedShareRequest(ref=cs3scr.ShareReference(key=share_key)) else: raise ValueError("opaque_id or share_key is required") - res = self._gateway.GetReceivedShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.GetReceivedShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "get received share", f'opaque_id/share_key="{opaque_id if opaque_id else share_key}"' ) @@ -259,11 +273,12 @@ def get_received_share(self, opaque_id: str = None, share_key: cs3scr.ShareKey = return res.share def update_received_share( - self, received_share: cs3scr.ReceivedShare, state: str = "SHARE_STATE_ACCEPTED" + self, auth_token: tuple, received_share: cs3scr.ReceivedShare, state: str = "SHARE_STATE_ACCEPTED" ) -> cs3scr.ReceivedShare: """ Update the state of a received share (SHARE_STATE_ACCEPTED, SHARE_STATE_ACCEPTED, SHARE_STATE_REJECTED). + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param recieved_share: ReceivedShare object. :param state: Share state to update to, defaults to SHARE_STATE_ACCEPTED, (REQUIRED). :return: Updated ReceivedShare object. @@ -283,7 +298,7 @@ def update_received_share( ), update_mask=field_masks.FieldMask(paths=["state"]), ) - res = self._gateway.UpdateReceivedShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.UpdateReceivedShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "update received share", f'opaque_id="{received_share.share.id.opaque_id}"' ) @@ -295,6 +310,7 @@ def update_received_share( def create_public_share( self, + auth_token: tuple, resource_info: cs3spr.ResourceInfo, role: str, password: str = None, @@ -307,6 +323,7 @@ def create_public_share( """ Create a public share. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource_info: Resource info, see file.stat (REQUIRED). :param role: Role to assign to the grantee, VIEWER or EDITOR (REQUIRED) :param password: Password to access the share. @@ -335,17 +352,18 @@ def create_public_share( notify_uploads_extra_recipients=notify_uploads_extra_recipients, ) - res = self._gateway.CreatePublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.CreatePublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "create public share", f'resource_id="{resource_info.id}"') self._log.debug(f'msg="Invoked CreatePublicShare" resource_id="{resource_info.id}" trace="{res.status.trace}"') return res.share def list_existing_public_shares( - self, filter_list: list = None, page_size: int = 0, page_token: str = None, sign: bool = None + self, auth_token: tuple, filter_list: list = None, page_size: int = 0, page_token: str = None, sign: bool = None ) -> list: """ List existing public shares. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter object to filter the shares, see create_public_share_filter. :param page_size: Number of shares to return in a page, defaults to 0 and then the server decides. :param page_token: Token to get to a specific page. @@ -358,7 +376,7 @@ def list_existing_public_shares( req = cs3slapi.ListPublicSharesRequest( filters=filter_list, page_size=page_size, page_token=page_token, sign=sign ) - res = self._gateway.ListExistingPublicShares(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.ListExistingPublicShares(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "list existing public shares", f'filter="{filter_list}"') self._log.debug( f'msg="Invoked ListExistingPublicShares" filter="{filter_list}" res_count="{len(res.share_infos)}" ' @@ -366,10 +384,13 @@ def list_existing_public_shares( ) return (res.share_infos, res.next_page_token) - def get_public_share(self, opaque_id: str = None, token: str = None, sign: bool = False) -> cs3slr.PublicShare: + def get_public_share( + self, auth_token: tuple, opaque_id: str = None, token: str = None, sign: bool = False + ) -> cs3slr.PublicShare: """ Get a public share by its opaque id or token, one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param opaque_id: Opaque share id (SEMI-OPTIONAL). :param share_token: Share token (SEMI-OPTIONAL). :param sign: if the signature should be included in the share. @@ -387,7 +408,7 @@ def get_public_share(self, opaque_id: str = None, token: str = None, sign: bool else: raise ValueError("token or opaque_id is required") req = cs3slapi.GetPublicShareRequest(ref=ref, sign=sign) - res = self._gateway.GetPublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.GetPublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "get public share", f'opaque_id/token="{opaque_id if opaque_id else token}"' ) @@ -399,6 +420,7 @@ def get_public_share(self, opaque_id: str = None, token: str = None, sign: bool def update_public_share( self, + auth_token: tuple, type: str, role: str, opaque_id: str = None, @@ -415,6 +437,7 @@ def update_public_share( however, other parameters are optional. Note that only the type of update specified will be applied. The role will only change if type is TYPE_PERMISSIONS. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param type: Type of update to perform TYPE_PERMISSIONS, TYPE_PASSWORD, TYPE_EXPIRATION, TYPE_DISPLAYNAME, TYPE_DESCRIPTION, TYPE_NOTIFYUPLOADS, TYPE_NOTIFYUPLOADSEXTRARECIPIENTS (REQUIRED). :param role: Role to assign to the grantee, VIEWER or EDITOR (REQUIRED). @@ -451,7 +474,7 @@ def update_public_share( password=password, ) req = cs3slapi.UpdatePublicShareRequest(ref=ref, update=update) - res = self._gateway.UpdatePublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.UpdatePublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "update public share", @@ -463,10 +486,11 @@ def update_public_share( ) return res.share - def remove_public_share(self, token: str = None, opaque_id: str = None) -> None: + def remove_public_share(self, auth_token: tuple, token: str = None, opaque_id: str = None) -> None: """ Remove a public share by its token or opaque id, one of them is required. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param token: Share token (SEMI-OPTIONAL). :param opaque_id: Opaque share id (SEMI-OPTIONAL). :return: None @@ -481,7 +505,7 @@ def remove_public_share(self, token: str = None, opaque_id: str = None) -> None: raise ValueError("token or opaque_id is required") req = cs3slapi.RemovePublicShareRequest(ref=ref) - res = self._gateway.RemovePublicShare(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.RemovePublicShare(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors( res.status, "remove public share", f'opaque_id/token="{opaque_id if opaque_id else token}"' ) diff --git a/src/statuscodehandler.py b/src/statuscodehandler.py index 2a63907..770af59 100644 --- a/src/statuscodehandler.py +++ b/src/statuscodehandler.py @@ -3,16 +3,17 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -from exceptions.exceptions import AuthenticationException, NotFoundException, \ - UnknownException, AlreadyExistsException, FileLockedException, UnimplementedException import logging -from config import Config import cs3.rpc.v1beta1.code_pb2 as cs3code import cs3.rpc.v1beta1.status_pb2 as cs3status +from exceptions.exceptions import AuthenticationException, NotFoundException, \ + UnknownException, AlreadyExistsException, FileLockedException, UnimplementedException +from config import Config + class StatusCodeHandler: def __init__(self, log: logging.Logger, config: Config) -> None: diff --git a/src/user.py b/src/user.py index 2d64d4b..83ae5b2 100644 --- a/src/user.py +++ b/src/user.py @@ -3,15 +3,15 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ import logging -from auth import Auth -from config import Config import cs3.identity.user.v1beta1.resources_pb2 as cs3iur import cs3.identity.user.v1beta1.user_api_pb2 as cs3iu from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub + +from config import Config from statuscodehandler import StatusCodeHandler @@ -25,7 +25,6 @@ def __init__( config: Config, log: logging.Logger, gateway: GatewayAPIStub, - auth: Auth, status_code_handler: StatusCodeHandler, ) -> None: """ @@ -35,7 +34,6 @@ def __init__( :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. :param auth: An instance of the auth class. """ - self._auth: Auth = auth self._log: logging.Logger = log self._gateway: GatewayAPIStub = gateway self._config: Config = config @@ -92,10 +90,11 @@ def get_user_groups(self, idp, opaque_id) -> list[str]: self._log.debug(f'msg="Invoked GetUserGroups" opaque_id="{opaque_id}" trace="{res.status.trace}"') return res.groups - def find_users(self, filter) -> list[cs3iur.User]: + def find_users(self, auth_token: tuple, filter) -> list[cs3iur.User]: """ Find a user based on a filter. + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param filter: Filter to search for. :return: a list of user(s). :raises: NotFoundException (User not found) @@ -103,7 +102,7 @@ def find_users(self, filter) -> list[cs3iur.User]: :raises: UnknownException (Unknown error) """ req = cs3iu.FindUsersRequest(filter=filter, skip_fetching_user_groups=True) - res = self._gateway.FindUsers(request=req, metadata=[self._auth.get_token()]) + res = self._gateway.FindUsers(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "find users") self._log.debug(f'msg="Invoked FindUsers" filter="{filter}" trace="{res.status.trace}"') return res.users diff --git a/tests/fixtures.py b/tests/fixtures.py index 075f03e..4ac6312 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -5,11 +5,10 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch from configparser import ConfigParser @@ -17,16 +16,15 @@ import base64 import json -sys.path.append("src/") -from cs3client import CS3Client # noqa: E402 -from file import File # noqa: E402 -from auth import Auth # noqa: E402 -from user import User # noqa: E402 -from statuscodehandler import StatusCodeHandler # noqa: E402 -from share import Share # noqa: E402 -from app import App # noqa: E402 -from checkpoint import Checkpoint # noqa: E402 -from config import Config # noqa: E402 +from cs3client import CS3Client +from file import File +from auth import Auth +from user import User +from statuscodehandler import StatusCodeHandler +from share import Share +from app import App +from checkpoint import Checkpoint +from config import Config @pytest.fixture @@ -99,24 +97,15 @@ def mock_gateway(mock_gateway_stub_class): return mock_gateway_stub -# All the parameters are inferred by pytest from existing fixtures -@pytest.fixture -def mock_authentication(mock_gateway, mock_config, mock_logger): - # Set up mock response for Authenticate method - mock_authentication = Auth(Config(mock_config, "cs3client"), mock_logger, mock_gateway) - mock_authentication.set_client_secret("test") - return mock_authentication - - # Here the order of patches correspond to the parameters of the function # (patches are applied from the bottom up) # and the last two parameters are inferred by pytest from existing fixtures @pytest.fixture -@patch("cs3client.grpc.secure_channel", autospec=True) -@patch("cs3client.grpc.channel_ready_future", autospec=True) -@patch("cs3client.grpc.insecure_channel", autospec=True) -@patch("cs3client.cs3gw_grpc.GatewayAPIStub", autospec=True) -@patch("cs3client.grpc.ssl_channel_credentials", autospec=True) +@patch("cs3client.cs3client.grpc.secure_channel", autospec=True) +@patch("cs3client.cs3client.grpc.channel_ready_future", autospec=True) +@patch("cs3client.cs3client.grpc.insecure_channel", autospec=True) +@patch("cs3client.cs3client.cs3gw_grpc.GatewayAPIStub", autospec=True) +@patch("cs3client.cs3client.grpc.ssl_channel_credentials", autospec=True) def cs3_client_secure( mock_ssl_channel_credentials, mock_gateway_stub_class, @@ -143,11 +132,11 @@ def cs3_client_secure( # (patches are applied from the bottom up) # and the last two parameters are inferred by pytest from existing fixtures @pytest.fixture -@patch("cs3client.grpc.secure_channel") -@patch("cs3client.grpc.insecure_channel") -@patch("cs3client.grpc.channel_ready_future") -@patch("cs3client.cs3gw_grpc.GatewayAPIStub") -@patch("cs3client.grpc.ssl_channel_credentials") +@patch("cs3client.cs3client.grpc.secure_channel") +@patch("cs3client.cs3client.grpc.insecure_channel") +@patch("cs3client.cs3client.grpc.channel_ready_future") +@patch("cs3client.cs3client.cs3gw_grpc.GatewayAPIStub") +@patch("cs3client.cs3client.grpc.ssl_channel_credentials") def cs3_client_insecure( mock_ssl_channel_credentials, mock_gateway_stub_class, @@ -171,28 +160,36 @@ def cs3_client_insecure( @pytest.fixture -def app_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def auth_instance(cs3_client_insecure): + # Set up mock response for Authenticate method + auth = Auth(cs3_client_insecure) + auth.set_client_secret("test") + return auth + + +# All the parameters are inferred by pytest from existing fixtures +@pytest.fixture +def app_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): app = App( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return app @pytest.fixture -def checkpoint_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def checkpoint_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): checkpoint = Checkpoint( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return checkpoint @pytest.fixture -def share_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def share_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): share = Share( Config(mock_config, "cs3client"), mock_logger, mock_gateway, - mock_authentication, mock_status_code_handler, ) return share @@ -200,16 +197,16 @@ def share_instance(mock_authentication, mock_gateway, mock_config, mock_logger, # All parameters are inferred by pytest from existing fixtures @pytest.fixture -def file_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def file_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): file = File( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return file @pytest.fixture -def user_instance(mock_authentication, mock_gateway, mock_config, mock_logger, mock_status_code_handler): +def user_instance(mock_gateway, mock_config, mock_logger, mock_status_code_handler): user = User( - Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_authentication, mock_status_code_handler + Config(mock_config, "cs3client"), mock_logger, mock_gateway, mock_status_code_handler ) return user diff --git a/tests/test_app.py b/tests/test_app.py index 8e3cf93..b2da006 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -5,34 +5,27 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys import cs3.rpc.v1beta1.code_pb2 as cs3code from unittest.mock import Mock, patch import pytest - -sys.path.append("src/") - -from exceptions.exceptions import ( # noqa: E402 +from exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) -from cs3resource import Resource # noqa: E402 - -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from cs3resource import Resource +from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, app_instance, mock_status_code_handler, ) # Test cases for the App class -# Test cases for the App class `list_app_providers` method using parameterized tests @pytest.mark.parametrize( @@ -50,13 +43,14 @@ def test_list_app_providers( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.providers = providers + auth_token = ('x-access-token', "some_token") with patch.object(app_instance._gateway, "ListAppProviders", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - app_instance.list_app_providers() + app_instance.list_app_providers(auth_token) else: - result = app_instance.list_app_providers() + result = app_instance.list_app_providers(auth_token) assert result == providers @@ -80,11 +74,12 @@ def test_open_in_app( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.OpenInAppURL = open_in_app_url + auth_token = ('x-access-token', "some_token") with patch.object(app_instance._gateway, "OpenInApp", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - app_instance.open_in_app(resource, view_mode, app) + app_instance.open_in_app(auth_token, resource, view_mode, app) else: - result = app_instance.open_in_app(resource, view_mode, app) + result = app_instance.open_in_app(auth_token, resource, view_mode, app) assert result == open_in_app_url diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 20c679d..110808c 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -5,34 +5,26 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys from unittest.mock import Mock, patch import pytest import cs3.rpc.v1beta1.code_pb2 as cs3code - -sys.path.append("src/") - -from exceptions.exceptions import ( # noqa: E402 +from exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) - -from cs3resource import Resource # noqa: E402 - -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from cs3resource import Resource +from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, checkpoint_instance, mock_status_code_handler, ) - # Test cases for the Checkpoint class @@ -56,13 +48,14 @@ def test_list_file_versions( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.versions = versions + auth_token = ('x-access-token', "some_token") with patch.object(checkpoint_instance._gateway, "ListFileVersions", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - checkpoint_instance.list_file_versions(resource, page_token, page_size) + checkpoint_instance.list_file_versions(auth_token, resource, page_token, page_size) else: - result = checkpoint_instance.list_file_versions(resource, page_token, page_size) + result = checkpoint_instance.list_file_versions(auth_token, resource, page_token, page_size) assert result == versions @@ -85,11 +78,12 @@ def test_restore_file_version( mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(checkpoint_instance._gateway, "RestoreFileVersion", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - checkpoint_instance.restore_file_version(resource, version_key, lock_id) + checkpoint_instance.restore_file_version(auth_token, resource, version_key, lock_id) else: - result = checkpoint_instance.restore_file_version(resource, version_key, lock_id) + result = checkpoint_instance.restore_file_version(auth_token, resource, version_key, lock_id) assert result is None diff --git a/tests/test_cs3client.py b/tests/test_cs3client.py index 2aa169c..630cd27 100644 --- a/tests/test_cs3client.py +++ b/tests/test_cs3client.py @@ -5,13 +5,9 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 26/07/2024 +Last updated: 30/08/2024 """ -import sys - -sys.path.append("src/") - from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) cs3_client_insecure, cs3_client_secure, @@ -21,6 +17,8 @@ create_mock_jwt, ) +# Test cases for the cs3client class. + def test_cs3client_initialization_secure(cs3_client_secure): # noqa: F811 (not a redefinition) client = cs3_client_secure @@ -45,17 +43,10 @@ def test_cs3client_initialization_secure(cs3_client_secure): # noqa: F811 (not # Make sure the gRPC channel is correctly created assert client.channel is not None assert client._gateway is not None - assert client.auth is not None assert client.file is not None - # Make sure auth objects are correctly set - assert client.auth._gateway is not None - assert client.auth._config is not None - assert client.auth._log is not None - # Make sure file objects are correctly set assert client.file._gateway is not None - assert client.file._auth is not None assert client.file._config is not None assert client.file._log is not None @@ -83,16 +74,9 @@ def test_cs3client_initialization_insecure(cs3_client_insecure): # noqa: F811 ( # Make sure the gRPC channel is correctly created assert client.channel is not None assert client._gateway is not None - assert client.auth is not None assert client.file is not None - # Make sure auth objects are correctly set - assert client.auth._gateway is not None - assert client.auth._config is not None - assert client.auth._log is not None - # Make sure file objects are correctly set assert client.file._gateway is not None - assert client.file._auth is not None assert client.file._config is not None assert client.file._log is not None diff --git a/tests/test_file.py b/tests/test_file.py index 24d0a94..218772f 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -5,32 +5,29 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 30/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch import cs3.rpc.v1beta1.code_pb2 as cs3code - -sys.path.append("src/") - -from cs3resource import Resource # noqa: E402 -from exceptions.exceptions import ( # noqa: E402 +from cs3resource import Resource +from exceptions.exceptions import ( AuthenticationException, NotFoundException, FileLockedException, UnknownException, ) -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, file_instance, mock_status_code_handler, ) +# Test cases for the file class. + @pytest.mark.parametrize( "status_code, status_message, expected_exception, expected_result", @@ -48,15 +45,16 @@ def test_stat( mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") if status_code == cs3code.CODE_OK: mock_response.info = status_message with patch.object(file_instance._gateway, "Stat", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.stat(resource) + file_instance.stat(auth_token, resource) else: - result = file_instance.stat(resource) + result = file_instance.stat(auth_token, resource) assert result == expected_result @@ -76,13 +74,14 @@ def test_set_xattr(file_instance, status_code, status_message, expected_exceptio mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "SetArbitraryMetadata", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.set_xattr(resource, key, value) + file_instance.set_xattr(auth_token, resource, key, value) else: - file_instance.set_xattr(resource, key, value) + file_instance.set_xattr(auth_token, resource, key, value) @pytest.mark.parametrize( @@ -103,13 +102,14 @@ def test_remove_xattr( mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "UnsetArbitraryMetadata", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.remove_xattr(resource, key) + file_instance.remove_xattr(auth_token, resource, key) else: - file_instance.remove_xattr(resource, key) + file_instance.remove_xattr(auth_token, resource, key) @pytest.mark.parametrize( @@ -129,13 +129,14 @@ def test_rename_file(file_instance, status_code, status_message, expected_except mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "Move", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.rename_file(resource, newresource) + file_instance.rename_file(auth_token, resource, newresource) else: - file_instance.rename_file(resource, newresource) + file_instance.rename_file(auth_token, resource, newresource) @pytest.mark.parametrize( @@ -152,13 +153,14 @@ def test_remove_file(file_instance, status_code, status_message, expected_except mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "Delete", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.remove_file(resource) + file_instance.remove_file(auth_token, resource) else: - file_instance.remove_file(resource) + file_instance.remove_file(auth_token, resource) @pytest.mark.parametrize( @@ -175,13 +177,14 @@ def test_touch_file(file_instance, status_code, status_message, expected_excepti mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "TouchFile", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.touch_file(resource) + file_instance.touch_file(auth_token, resource) else: - file_instance.touch_file(resource) + file_instance.touch_file(auth_token, resource) @pytest.mark.parametrize( @@ -208,6 +211,7 @@ def test_write_file( mock_upload_response.status.code = status_code mock_upload_response.status.message = status_message mock_upload_response.protocols = [Mock(protocol="simple", upload_endpoint="http://example.com", token="token")] + auth_token = ('x-access-token', "some_token") mock_put_response = Mock() mock_put_response.status_code = put_response_status @@ -215,10 +219,10 @@ def test_write_file( with patch.object(file_instance._gateway, "InitiateFileUpload", return_value=mock_upload_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.write_file(resource, content, size) + file_instance.write_file(auth_token, resource, content, size) else: with patch("requests.put", return_value=mock_put_response): - file_instance.write_file(resource, content, size) + file_instance.write_file(auth_token, resource, content, size) @pytest.mark.parametrize( @@ -235,13 +239,14 @@ def test_make_dir(file_instance, status_code, status_message, expected_exception mock_response = Mock() mock_response.status.code = status_code mock_response.status.message = status_message + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "CreateContainer", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - file_instance.make_dir(resource) + file_instance.make_dir(auth_token, resource) else: - file_instance.make_dir(resource) + file_instance.make_dir(auth_token, resource) @pytest.mark.parametrize( @@ -262,18 +267,19 @@ def test_list_dir( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.infos = infos + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "ListContainer", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - res = file_instance.list_dir(resource) + res = file_instance.list_dir(auth_token, resource) # Lazy evaluation first_item = next(res, None) if first_item is not None: for _ in res: pass else: - res = file_instance.list_dir(resource) + res = file_instance.list_dir(auth_token, resource) # Lazy evaluation first_item = next(res, None) assert first_item == "file1" @@ -303,11 +309,12 @@ def test_read_file( mock_fileget_response = Mock() mock_fileget_response.status_code = 200 mock_fileget_response.iter_content = Mock(return_value=iter_content) + auth_token = ('x-access-token', "some_token") with patch.object(file_instance._gateway, "InitiateFileDownload", return_value=mock_download_response): if expected_exception: with pytest.raises(expected_exception): - res = file_instance.read_file(resource) + res = file_instance.read_file(auth_token, resource) # Lazy evaluation first_item = next(res, None) if first_item is not None: @@ -315,7 +322,7 @@ def test_read_file( pass else: with patch("requests.get", return_value=mock_fileget_response): - res = file_instance.read_file(resource) + res = file_instance.read_file(auth_token, resource) # Lazy evaluation chunks = list(res) assert chunks == iter_content diff --git a/tests/test_resource.py b/tests/test_resource.py index 4ce89c2..32ea213 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -8,13 +8,9 @@ Last updated: 26/07/2024 """ -import sys import unittest import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr - -sys.path.append("src/") - -from cs3resource import Resource # noqa: E402 +from cs3resource import Resource class TestResource(unittest.TestCase): diff --git a/tests/test_share.py b/tests/test_share.py index 9a80add..432d86b 100644 --- a/tests/test_share.py +++ b/tests/test_share.py @@ -5,36 +5,31 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 28/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch import cs3.sharing.collaboration.v1beta1.resources_pb2 as cs3scr import cs3.sharing.link.v1beta1.link_api_pb2 as cs3slapi import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.rpc.v1beta1.code_pb2 as cs3code - -sys.path.append("src/") -from exceptions.exceptions import ( # noqa: E402 +from exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, FileLockedException, AlreadyExistsException, ) -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, share_instance, mock_status_code_handler, ) - -# Test cases for the Share class `get_share` method using parameterized tests +# Test cases for the Share class. @pytest.mark.parametrize( @@ -63,16 +58,27 @@ def test_create_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = status_message + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "CreateShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): share_instance.create_share( - resource_info=resource_info, opaque_id=opaque_id, idp=idp, role=role, grantee_type=grantee_type + auth_token, + resource_info=resource_info, + opaque_id=opaque_id, + idp=idp, + role=role, + grantee_type=grantee_type ) else: result = share_instance.create_share( - resource_info=resource_info, opaque_id=opaque_id, idp=idp, role=role, grantee_type=grantee_type + auth_token, + resource_info=resource_info, + opaque_id=opaque_id, + idp=idp, + role=role, + grantee_type=grantee_type ) assert result == expected_result @@ -94,13 +100,14 @@ def test_list_existing_shares( if status_code == cs3code.CODE_OK: mock_response.share_infos = expected_result[0] mock_response.next_page_token = expected_result[1] + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "ListExistingShares", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.list_existing_shares() + share_instance.list_existing_shares(auth_token) else: - result = share_instance.list_existing_shares() + result = share_instance.list_existing_shares(auth_token) assert result == expected_result @@ -123,13 +130,14 @@ def test_get_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "GetShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.get_share(share_id) + share_instance.get_share(auth_token, share_id) else: - result = share_instance.get_share(share_id) + result = share_instance.get_share(auth_token, share_id) assert result == expected_result @@ -152,13 +160,14 @@ def test_remove_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "RemoveShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.remove_share(share_id) + share_instance.remove_share(auth_token, share_id) else: - result = share_instance.remove_share(share_id) + result = share_instance.remove_share(auth_token, share_id) assert result is None @@ -183,13 +192,14 @@ def test_update_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "UpdateShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.update_share(role=role, opaque_id=opaque_id) + share_instance.update_share(auth_token, role=role, opaque_id=opaque_id) else: - result = share_instance.update_share(role=role, opaque_id=opaque_id) + result = share_instance.update_share(auth_token, role=role, opaque_id=opaque_id) assert result == expected_result @@ -211,13 +221,14 @@ def test_list_existing_received_shares( if status_code == cs3code.CODE_OK: mock_response.share_infos = expected_result[0] mock_response.next_page_token = expected_result[1] + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "ListExistingReceivedShares", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.list_received_existing_shares() + share_instance.list_received_existing_shares(auth_token) else: - result = share_instance.list_received_existing_shares() + result = share_instance.list_received_existing_shares(auth_token) assert result == expected_result @@ -240,13 +251,14 @@ def test_get_received_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "GetReceivedShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.get_received_share(share_id) + share_instance.get_received_share(auth_token, share_id) else: - result = share_instance.get_received_share(share_id) + result = share_instance.get_received_share(auth_token, share_id) assert result == expected_result @@ -271,13 +283,14 @@ def test_update_received_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "UpdateReceivedShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.update_received_share(received_share=received_share) + share_instance.update_received_share(auth_token, received_share=received_share) else: - result = share_instance.update_received_share(received_share=received_share) + result = share_instance.update_received_share(auth_token, received_share=received_share) assert result == expected_result @@ -304,13 +317,14 @@ def test_create_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "CreatePublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.create_public_share(resource_info=resource_info, role=role) + share_instance.create_public_share(auth_token, resource_info=resource_info, role=role) else: - result = share_instance.create_public_share(resource_info=resource_info, role=role) + result = share_instance.create_public_share(auth_token, resource_info=resource_info, role=role) assert result == expected_result @@ -331,13 +345,14 @@ def list_existing_public_shares( if status_code == cs3code.CODE_OK: mock_response.share_infos = expected_result[0] mock_response.next_page_token = expected_result[1] + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "ListExistingPublicShares", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.list_existing_public_shares() + share_instance.list_existing_public_shares(auth_token) else: - result = share_instance.list_existing_public_shares() + result = share_instance.list_existing_public_shares(auth_token) assert result == expected_result @@ -360,13 +375,14 @@ def test_get_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "GetPublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.get_public_share(share_id) + share_instance.get_public_share(auth_token, share_id) else: - result = share_instance.get_public_share(share_id) + result = share_instance.get_public_share(auth_token, share_id) assert result == expected_result @@ -391,13 +407,14 @@ def test_update_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "UpdatePublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.update_public_share(type=type, role=role, opaque_id=opqaue_id) + share_instance.update_public_share(auth_token, type=type, role=role, opaque_id=opqaue_id) else: - result = share_instance.update_public_share(type=type, role=role, opaque_id=opqaue_id) + result = share_instance.update_public_share(auth_token, type=type, role=role, opaque_id=opqaue_id) assert result == expected_result @@ -423,13 +440,14 @@ def test_remove_public_share( mock_response.status.message = status_message if status_code == cs3code.CODE_OK: mock_response.share = expected_result + auth_token = ('x-access-token', "some_token") with patch.object(share_instance._gateway, "RemovePublicShare", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - share_instance.remove_public_share(share_id) + share_instance.remove_public_share(auth_token, share_id) else: - result = share_instance.remove_public_share(share_id) + result = share_instance.remove_public_share(auth_token, share_id) assert result is None diff --git a/tests/test_user.py b/tests/test_user.py index 8c51879..a02e5ea 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -5,32 +5,28 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 19/08/2024 +Last updated: 28/08/2024 """ -import sys import pytest from unittest.mock import Mock, patch import cs3.rpc.v1beta1.code_pb2 as cs3code - -sys.path.append("src/") - -from exceptions.exceptions import ( # noqa: E402 +from exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, - mock_authentication, mock_gateway, user_instance, mock_status_code_handler, ) - # Test cases for the User class + + @pytest.mark.parametrize( "status_code, status_message, expected_exception, user_data", [ @@ -133,11 +129,12 @@ def test_find_users( mock_response.status.code = status_code mock_response.status.message = status_message mock_response.users = users + auth_token = ('x-access-token', "some_token") with patch.object(user_instance._gateway, "FindUsers", return_value=mock_response): if expected_exception: with pytest.raises(expected_exception): - user_instance.find_users(filter) + user_instance.find_users(auth_token, filter) else: - result = user_instance.find_users(filter) + result = user_instance.find_users(auth_token, filter) assert result == users From 96fd8110898baa90e7da897593cf5be1a3143834 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Fri, 30 Aug 2024 17:55:53 +0200 Subject: [PATCH 2/8] Updated project structure --- .gitignore | 3 +++ README.md | 4 +-- {src => cs3client}/__init__.py | 0 {src => cs3client}/app.py | 6 ++--- {src => cs3client}/auth.py | 6 ++--- {src => cs3client}/checkpoint.py | 6 ++--- {src => cs3client}/config.py | 0 {src => cs3client}/cs3client.py | 14 +++++----- {src => cs3client}/cs3resource.py | 0 {src => cs3client}/exceptions/__init__.py | 0 {src => cs3client}/exceptions/exceptions.py | 0 {src => cs3client}/file.py | 10 +++---- {src => cs3client}/share.py | 6 ++--- {src => cs3client}/statuscodehandler.py | 4 +-- {src => cs3client}/user.py | 4 +-- docs/.DS_Store | Bin 6148 -> 0 bytes examples/app_api_example.py | 6 ++--- examples/auth_example.py | 4 +-- examples/checkpoints_api_example.py | 6 ++--- examples/file_api_example.py | 6 ++--- examples/shares_api_example.py | 6 ++--- examples/user_api_example.py | 4 +-- setup.py | 17 +++++++++--- tests/__init__.py | 0 tests/fixtures.py | 28 +++++++------------- tests/test_app.py | 7 ++--- tests/test_checkpoint.py | 7 ++--- tests/test_cs3client.py | 2 +- tests/test_file.py | 7 ++--- tests/test_resource.py | 3 ++- tests/test_share.py | 5 ++-- tests/test_user.py | 5 ++-- 32 files changed, 92 insertions(+), 84 deletions(-) rename {src => cs3client}/__init__.py (100%) rename {src => cs3client}/app.py (96%) rename {src => cs3client}/auth.py (97%) rename {src => cs3client}/checkpoint.py (97%) rename {src => cs3client}/config.py (100%) rename {src => cs3client}/cs3client.py (92%) rename {src => cs3client}/cs3resource.py (100%) rename {src => cs3client}/exceptions/__init__.py (100%) rename {src => cs3client}/exceptions/exceptions.py (100%) rename {src => cs3client}/file.py (98%) rename {src => cs3client}/share.py (99%) rename {src => cs3client}/statuscodehandler.py (97%) rename {src => cs3client}/user.py (98%) delete mode 100644 docs/.DS_Store create mode 100644 tests/__init__.py diff --git a/.gitignore b/.gitignore index 34de50d..4762aba 100644 --- a/.gitignore +++ b/.gitignore @@ -157,6 +157,9 @@ dmypy.json # Cython debug symbols cython_debug/ +# MacOS +.DS_Store + # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore diff --git a/README.md b/README.md index 388a33c..3148378 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,8 @@ To use `cs3client`, you first need to import and configure it. Here's a simple e ```python import logging import configparser -from cs3client import CS3Client -from auth import Auth +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth config = configparser.ConfigParser() with open("default.conf") as fdef: diff --git a/src/__init__.py b/cs3client/__init__.py similarity index 100% rename from src/__init__.py rename to cs3client/__init__.py diff --git a/src/app.py b/cs3client/app.py similarity index 96% rename from src/app.py rename to cs3client/app.py index d10e418..ea29c07 100644 --- a/src/app.py +++ b/cs3client/app.py @@ -13,9 +13,9 @@ import cs3.app.provider.v1beta1.resources_pb2 as cs3apr from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub -from cs3resource import Resource -from statuscodehandler import StatusCodeHandler -from config import Config +from .cs3resource import Resource +from .statuscodehandler import StatusCodeHandler +from .config import Config class App: diff --git a/src/auth.py b/cs3client/auth.py similarity index 97% rename from src/auth.py rename to cs3client/auth.py index b8da76d..dd1d9cf 100644 --- a/src/auth.py +++ b/cs3client/auth.py @@ -15,9 +15,9 @@ from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub from cs3.rpc.v1beta1.code_pb2 import CODE_OK -from cs3client import CS3Client -from exceptions.exceptions import AuthenticationException, SecretNotSetException -from config import Config +from .cs3client import CS3Client +from .exceptions.exceptions import AuthenticationException, SecretNotSetException +from .config import Config class Auth: diff --git a/src/checkpoint.py b/cs3client/checkpoint.py similarity index 97% rename from src/checkpoint.py rename to cs3client/checkpoint.py index ea9bcca..dfedb00 100644 --- a/src/checkpoint.py +++ b/cs3client/checkpoint.py @@ -12,9 +12,9 @@ import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3spp from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub -from config import Config -from statuscodehandler import StatusCodeHandler -from cs3resource import Resource +from .config import Config +from .statuscodehandler import StatusCodeHandler +from .cs3resource import Resource class Checkpoint: diff --git a/src/config.py b/cs3client/config.py similarity index 100% rename from src/config.py rename to cs3client/config.py diff --git a/src/cs3client.py b/cs3client/cs3client.py similarity index 92% rename from src/cs3client.py rename to cs3client/cs3client.py index cfaf301..79cb608 100644 --- a/src/cs3client.py +++ b/cs3client/cs3client.py @@ -11,13 +11,13 @@ import cs3.gateway.v1beta1.gateway_api_pb2_grpc as cs3gw_grpc from configparser import ConfigParser -from file import File -from user import User -from share import Share -from statuscodehandler import StatusCodeHandler -from app import App -from checkpoint import Checkpoint -from config import Config +from .file import File +from .user import User +from .share import Share +from .statuscodehandler import StatusCodeHandler +from .app import App +from .checkpoint import Checkpoint +from .config import Config class CS3Client: diff --git a/src/cs3resource.py b/cs3client/cs3resource.py similarity index 100% rename from src/cs3resource.py rename to cs3client/cs3resource.py diff --git a/src/exceptions/__init__.py b/cs3client/exceptions/__init__.py similarity index 100% rename from src/exceptions/__init__.py rename to cs3client/exceptions/__init__.py diff --git a/src/exceptions/exceptions.py b/cs3client/exceptions/exceptions.py similarity index 100% rename from src/exceptions/exceptions.py rename to cs3client/exceptions/exceptions.py diff --git a/src/file.py b/cs3client/file.py similarity index 98% rename from src/file.py rename to cs3client/file.py index 2201f57..43f65e8 100644 --- a/src/file.py +++ b/cs3client/file.py @@ -3,7 +3,7 @@ Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch -Last updated: 29/08/2024 +Last updated: 30/08/2024 """ import time @@ -16,10 +16,10 @@ from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub import cs3.types.v1beta1.types_pb2 as types -from config import Config -from exceptions.exceptions import AuthenticationException, FileLockedException -from cs3resource import Resource -from statuscodehandler import StatusCodeHandler +from .config import Config +from .exceptions.exceptions import AuthenticationException, FileLockedException +from .cs3resource import Resource +from .statuscodehandler import StatusCodeHandler class File: diff --git a/src/share.py b/cs3client/share.py similarity index 99% rename from src/share.py rename to cs3client/share.py index cc86819..da401a3 100644 --- a/src/share.py +++ b/cs3client/share.py @@ -18,9 +18,9 @@ import google.protobuf.field_mask_pb2 as field_masks import cs3.types.v1beta1.types_pb2 as cs3types -from cs3resource import Resource -from config import Config -from statuscodehandler import StatusCodeHandler +from .cs3resource import Resource +from .config import Config +from .statuscodehandler import StatusCodeHandler class Share: diff --git a/src/statuscodehandler.py b/cs3client/statuscodehandler.py similarity index 97% rename from src/statuscodehandler.py rename to cs3client/statuscodehandler.py index 770af59..47b48a5 100644 --- a/src/statuscodehandler.py +++ b/cs3client/statuscodehandler.py @@ -10,9 +10,9 @@ import cs3.rpc.v1beta1.code_pb2 as cs3code import cs3.rpc.v1beta1.status_pb2 as cs3status -from exceptions.exceptions import AuthenticationException, NotFoundException, \ +from .exceptions.exceptions import AuthenticationException, NotFoundException, \ UnknownException, AlreadyExistsException, FileLockedException, UnimplementedException -from config import Config +from .config import Config class StatusCodeHandler: diff --git a/src/user.py b/cs3client/user.py similarity index 98% rename from src/user.py rename to cs3client/user.py index 83ae5b2..3c4b03d 100644 --- a/src/user.py +++ b/cs3client/user.py @@ -11,8 +11,8 @@ import cs3.identity.user.v1beta1.user_api_pb2 as cs3iu from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub -from config import Config -from statuscodehandler import StatusCodeHandler +from .config import Config +from .statuscodehandler import StatusCodeHandler class User: diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index eb02f0b49f7e2ec103e5c6c5ea55d4cfc6475261..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOH0E*5Z>*>ZYe?z3Oz1(E!c-hi+zJyZy;aV>cCNrH7g%b$5yNQxeEru#f!nw}%j6>8#ee5)s%Wl8llD*Dg)so9W zx80H_z0=jIE)I{5&o0N$@k^rK43Qid*RpT1fOk;VDth&1Nvx7buof9bBq1?C3=jjv zz-}^N&jZoe&GM;gVt^Ry()=Fp(&`>ZgM+F4*jY|Lwa35)Er}0bFA=1.47.0", @@ -37,4 +35,15 @@ "protobuf", "cryptography", ], + license="Apache 2.0", + classifiers=[ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Operating System :: OS Independent", + ], ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures.py b/tests/fixtures.py index 4ac6312..6ef39dd 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -12,19 +12,18 @@ import pytest from unittest.mock import Mock, patch from configparser import ConfigParser -import cs3.rpc.v1beta1.code_pb2 as cs3code import base64 import json -from cs3client import CS3Client -from file import File -from auth import Auth -from user import User -from statuscodehandler import StatusCodeHandler -from share import Share -from app import App -from checkpoint import Checkpoint -from config import Config +from cs3client.cs3client import CS3Client +from cs3client.file import File +from cs3client.auth import Auth +from cs3client.user import User +from cs3client.statuscodehandler import StatusCodeHandler +from cs3client.share import Share +from cs3client.app import App +from cs3client.checkpoint import Checkpoint +from cs3client.config import Config @pytest.fixture @@ -87,13 +86,6 @@ def mock_status_code_handler(mock_logger, mock_config): def mock_gateway(mock_gateway_stub_class): mock_gateway_stub = Mock() mock_gateway_stub_class.return_value = mock_gateway_stub - # Set up mock response for Authenticate method - mocked_token = create_mock_jwt() - mock_authenticate_response = Mock() - mock_authenticate_response.status.code = cs3code.CODE_OK - mock_authenticate_response.status.message = "" - mock_authenticate_response.token = mocked_token - mock_gateway_stub.Authenticate.return_value = mock_authenticate_response return mock_gateway_stub @@ -118,7 +110,6 @@ def cs3_client_secure( # Create CS3Client instance client = CS3Client(mock_config, "cs3client", mock_logger) - client.auth.set_client_secret("test") assert mock_secure_channel.called assert mock_channel_ready_future.called @@ -150,7 +141,6 @@ def cs3_client_insecure( # Create CS3Client instance client = CS3Client(mock_config, "cs3client", mock_logger) - client.auth.set_client_secret("test") assert mock_insecure_channel.called assert mock_channel_ready_future.called diff --git a/tests/test_app.py b/tests/test_app.py index b2da006..0233797 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -11,13 +11,14 @@ import cs3.rpc.v1beta1.code_pb2 as cs3code from unittest.mock import Mock, patch import pytest -from exceptions.exceptions import ( + +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) -from cs3resource import Resource -from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) +from cs3client.cs3resource import Resource +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, mock_gateway, diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 110808c..d005965 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -11,13 +11,14 @@ from unittest.mock import Mock, patch import pytest import cs3.rpc.v1beta1.code_pb2 as cs3code -from exceptions.exceptions import ( + +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) -from cs3resource import Resource -from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) +from cs3client.cs3resource import Resource +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, mock_gateway, diff --git a/tests/test_cs3client.py b/tests/test_cs3client.py index 630cd27..e4e6872 100644 --- a/tests/test_cs3client.py +++ b/tests/test_cs3client.py @@ -8,7 +8,7 @@ Last updated: 30/08/2024 """ -from fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401, E402 (they are used, the framework is not detecting it) cs3_client_insecure, cs3_client_secure, mock_config, diff --git a/tests/test_file.py b/tests/test_file.py index 218772f..7e83afa 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -11,14 +11,15 @@ import pytest from unittest.mock import Mock, patch import cs3.rpc.v1beta1.code_pb2 as cs3code -from cs3resource import Resource -from exceptions.exceptions import ( + +from cs3client.cs3resource import Resource +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, FileLockedException, UnknownException, ) -from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, mock_gateway, diff --git a/tests/test_resource.py b/tests/test_resource.py index 32ea213..59dd2b8 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -10,7 +10,8 @@ import unittest import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr -from cs3resource import Resource + +from cs3client.cs3resource import Resource class TestResource(unittest.TestCase): diff --git a/tests/test_share.py b/tests/test_share.py index 432d86b..295c928 100644 --- a/tests/test_share.py +++ b/tests/test_share.py @@ -14,14 +14,15 @@ import cs3.sharing.link.v1beta1.link_api_pb2 as cs3slapi import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.rpc.v1beta1.code_pb2 as cs3code -from exceptions.exceptions import ( + +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, FileLockedException, AlreadyExistsException, ) -from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, mock_gateway, diff --git a/tests/test_user.py b/tests/test_user.py index a02e5ea..837b183 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -11,12 +11,13 @@ import pytest from unittest.mock import Mock, patch import cs3.rpc.v1beta1.code_pb2 as cs3code -from exceptions.exceptions import ( + +from cs3client.exceptions.exceptions import ( AuthenticationException, NotFoundException, UnknownException, ) -from fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) +from .fixtures import ( # noqa: F401 (they are used, the framework is not detecting it) mock_config, mock_logger, mock_gateway, From 0d0d1815d03ff93bb338cc09d3b8bf3ac173c773 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Fri, 30 Aug 2024 17:58:57 +0200 Subject: [PATCH 3/8] Added build step to CI --- .github/workflows/ci-tests.yml | 9 ++++++--- .gitignore | 3 --- tests/fixtures.py | 7 +++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 45af396..456d92e 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Linting and unit tests +name: Lint, Build and Test on: push: branches: [ "main" ] @@ -32,8 +32,11 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide, we further relax this flake8 . --count --exit-zero --max-complexity=30 --max-line-length=130 --statistics - + - name: Build + run: | + pip install build + python -m build - name: Test with pytest run: | if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi - pytest --cov-report term --cov=src tests/ + pytest --cov-report term --cov=cs3client tests/ diff --git a/.gitignore b/.gitignore index 4762aba..34de50d 100644 --- a/.gitignore +++ b/.gitignore @@ -157,9 +157,6 @@ dmypy.json # Cython debug symbols cython_debug/ -# MacOS -.DS_Store - # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore diff --git a/tests/fixtures.py b/tests/fixtures.py index 6ef39dd..b70b26e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -14,6 +14,7 @@ from configparser import ConfigParser import base64 import json +import cs3.rpc.v1beta1.code_pb2 as cs3code from cs3client.cs3client import CS3Client from cs3client.file import File @@ -86,6 +87,12 @@ def mock_status_code_handler(mock_logger, mock_config): def mock_gateway(mock_gateway_stub_class): mock_gateway_stub = Mock() mock_gateway_stub_class.return_value = mock_gateway_stub + mocked_token = create_mock_jwt() + mock_authenticate_response = Mock() + mock_authenticate_response.status.code = cs3code.CODE_OK + mock_authenticate_response.status.message = "" + mock_authenticate_response.token = mocked_token + mock_gateway_stub.Authenticate.return_value = mock_authenticate_response return mock_gateway_stub From a6a5b9045b19b14fe69a7fd5651ff46fe4d9c968 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Fri, 30 Aug 2024 18:24:36 +0200 Subject: [PATCH 4/8] Added possibility to set client secret and id at both runtime and in config --- README.md | 8 ++++++-- cs3client/auth.py | 32 +++++++++++++++++++++-------- cs3client/config.py | 9 ++++++++ cs3client/statuscodehandler.py | 26 +++++++++++------------ examples/app_api_example.py | 2 ++ examples/auth_example.py | 2 ++ examples/checkpoints_api_example.py | 2 ++ examples/default.conf | 2 ++ examples/file_api_example.py | 2 ++ examples/shares_api_example.py | 2 ++ examples/user_api_example.py | 2 ++ 11 files changed, 65 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 3148378..669d715 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,14 @@ ssl_verify = False # Optional, defaults to an empty string ssl_client_cert = test_client_cert # Optional, defaults to an empty string -ssl_client_key = test_client_key +ssl_client_key = test_client_key # Optional, defaults to an empty string -ssl_ca_cert = test_ca_cert +ssl_ca_cert = test_ca_cert # Optinal, defaults to an empty string auth_client_id = einstein +# Optional (can also be set when instansiating the class) +auth_client_secret = relativity # Optional, defaults to basic auth_login_type = basic @@ -125,6 +127,8 @@ log = logging.getLogger(__name__) client = CS3Client(config, "cs3client", log) auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") # Set client secret (can also be set in config) auth.set_client_secret("") # Checks if token is expired if not return ('x-access-token', ) diff --git a/cs3client/auth.py b/cs3client/auth.py index dd1d9cf..56b4f89 100644 --- a/cs3client/auth.py +++ b/cs3client/auth.py @@ -37,20 +37,31 @@ def __init__(self, cs3_client: CS3Client) -> None: self._gateway: GatewayAPIStub = cs3_client._gateway self._log: logging.Logger = cs3_client._log self._config: Config = cs3_client._config - # The user should be able to change the client secret (e.g. token) at runtime - self._client_secret: str | None = None + # The user should be able to change the client secret (e.g. token) and client id at runtime + self._client_secret: str | None = self._config.auth_client_secret + self._client_id: str | None = self._config.auth_client_id self._token: str | None = None def set_client_secret(self, token: str) -> None: """ Sets the client secret, exists so that the user can change the client secret (e.g. token, password) at runtime, - without having to create a new Auth object. NOTE that token OR the client secret has to be set when - instantiating the client object. + without having to create a new Auth object. Note client secret has to be set when + instantiating the client object or through the configuration. :param token: Auth token/password. """ self._client_secret = token + def set_client_id(self, id: str) -> None: + """ + Sets the client id, exists so that the user can change the client id at runtime, without having to create + a new Auth object. Settings this (either through config or here) is optional unless you are using + basic authentication. + + :param token: id. + """ + self._client_id = id + def get_token(self) -> tuple[str, str]: """ Attempts to get a valid authentication token. If the token is not valid, a new token is requested @@ -72,19 +83,22 @@ def get_token(self) -> tuple[str, str]: # Token has expired or has not been set, obtain another one. req = AuthenticateRequest( type=self._config.auth_login_type, - client_id=self._config.auth_client_id, + client_id=self._client_id, client_secret=self._client_secret, ) # Send the authentication request to the CS3 Gateway res = self._gateway.Authenticate(req) if res.status.code != CODE_OK: - self._log.error(f"Failed to authenticate user {self._config.auth_client_id}, error: {res.status}") + self._log.error(f'msg="Failed to authenticate" ' + f'user="{self._client_id if self._client_id else "no_id_set"}" ' + f'error_code="{res.status}"') raise AuthenticationException( - f"Failed to authenticate user {self._config.auth_client_id}, error: {res.status}" + f'Failed to authenticate: user="{self._client_id if self._client_id else "no_id_set"}" ' + f'error_code="{res.status}"' ) self._token = res.token - self._log.debug(f'msg="Authenticated user" user="{self._config.auth_client_id}"') + self._log.debug(f'msg="Authenticated user" user="{self._client_id if self._client_id else "no_id_set"}"') return ("x-access-token", self._token) def list_auth_providers(self) -> list[str]: @@ -97,7 +111,7 @@ def list_auth_providers(self) -> list[str]: try: res = self._gateway.ListAuthProviders(request=ListAuthProvidersRequest()) if res.status.code != CODE_OK: - self._log.error(f"List auth providers request failed, error: {res.status}") + self._log.error(f'msg="List auth providers request failed" error_code="{res.status}"') raise Exception(res.status.message) except grpc.RpcError as e: self._log.error("List auth providers request failed") diff --git a/cs3client/config.py b/cs3client/config.py index 316b204..78ba99b 100644 --- a/cs3client/config.py +++ b/cs3client/config.py @@ -131,6 +131,15 @@ def auth_client_id(self) -> str: """ return self._config.get(self._config_category, "auth_client_id", fallback=None) + @property + def auth_client_secret(self) -> str: + """ + The auth_client_secret property returns the auth_client_secret value from the configuration, + + :return: auth_client_secret + """ + return self._config.get(self._config_category, "auth_client_secret", fallback=None) + @property def tus_enabled(self) -> bool: """ diff --git a/cs3client/statuscodehandler.py b/cs3client/statuscodehandler.py index 47b48a5..bed57f5 100644 --- a/cs3client/statuscodehandler.py +++ b/cs3client/statuscodehandler.py @@ -22,44 +22,44 @@ def __init__(self, log: logging.Logger, config: Config) -> None: def _log_not_found_info(self, status: cs3status.Status, operation: str, msg: str = None) -> None: self._log.info( - f'msg="Not found on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'msg="Not found on {operation}" {msg + " " if msg else ""} ' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' ) def _log_authentication_error(self, status: cs3status.Status, operation: str, msg: str = None) -> None: self._log.error( f'msg="Authentication failed on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' ) def _log_unknown_error(self, status: cs3status.Status, operation: str, msg: str = None) -> None: self._log.error( f'msg="Failed to {operation}, unknown error" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' ) def _log_precondition_info(self, status: cs3status.Status, operation: str, msg: str = None) -> None: self._log.info( f'msg="Failed precondition on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' ) def _log_already_exists(self, status: cs3status.Status, operation: str, msg: str = None) -> None: self._log.info( f'msg="Already exists on {operation}" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' ) def _log_unimplemented(self, status: cs3status.Status, operation: str, msg: str = None) -> None: self._log.info( f'msg="Invoked {operation} on unimplemented feature" {msg + " " if msg else ""}' - f'userid="{self._config.auth_client_id}" trace="{status.trace}" ' - f'reason="{status.message.replace('"', "'")}"' + f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' + f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' ) def handle_errors(self, status: cs3status.Status, operation: str, msg: str = None) -> None: diff --git a/examples/app_api_example.py b/examples/app_api_example.py index 7929a1c..608de6c 100644 --- a/examples/app_api_example.py +++ b/examples/app_api_example.py @@ -22,6 +22,8 @@ client = CS3Client(config, "cs3client", log) auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") # Set client secret (can also be set in config) auth.set_client_secret("") # Checks if token is expired if not return ('x-access-token', ) diff --git a/examples/auth_example.py b/examples/auth_example.py index e9c3db3..926c5c2 100644 --- a/examples/auth_example.py +++ b/examples/auth_example.py @@ -21,6 +21,8 @@ client = CS3Client(config, "cs3client", log) auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") # Set client secret (can also be set in config) auth.set_client_secret("") # Checks if token is expired if not return ('x-access-token', ) diff --git a/examples/checkpoints_api_example.py b/examples/checkpoints_api_example.py index d0871f0..7e2b9de 100644 --- a/examples/checkpoints_api_example.py +++ b/examples/checkpoints_api_example.py @@ -22,6 +22,8 @@ client = CS3Client(config, "cs3client", log) auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") # Set client secret (can also be set in config) auth.set_client_secret("") # Checks if token is expired if not return ('x-access-token', ) diff --git a/examples/default.conf b/examples/default.conf index e5be292..b8d0c68 100644 --- a/examples/default.conf +++ b/examples/default.conf @@ -33,6 +33,8 @@ ssl_ca_cert = test_ca_cert auth_client_id = einstein # Optional, defaults to basic auth_login_type = basic +# Optional (Can also be set after instantiating the Auth object) +auth_client_secret = relativity # For the future lock implementation diff --git a/examples/file_api_example.py b/examples/file_api_example.py index cd8c74f..db5fcdc 100644 --- a/examples/file_api_example.py +++ b/examples/file_api_example.py @@ -29,6 +29,8 @@ client = CS3Client(config, "cs3client", log) auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") # Set client secret (can also be set in config) auth.set_client_secret("") # Checks if token is expired if not return ('x-access-token', ) diff --git a/examples/shares_api_example.py b/examples/shares_api_example.py index 5085ee4..d3dd4f8 100644 --- a/examples/shares_api_example.py +++ b/examples/shares_api_example.py @@ -22,6 +22,8 @@ client = CS3Client(config, "cs3client", log) auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") # Set client secret (can also be set in config) auth.set_client_secret("") # Checks if token is expired if not return ('x-access-token', ) diff --git a/examples/user_api_example.py b/examples/user_api_example.py index ed0decb..7fdc105 100644 --- a/examples/user_api_example.py +++ b/examples/user_api_example.py @@ -21,6 +21,8 @@ client = CS3Client(config, "cs3client", log) auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") # Set client secret (can also be set in config) auth.set_client_secret("") # Checks if token is expired if not return ('x-access-token', ) From 005f12894a0c688c35f0e6bd41f32b50e98fdfe7 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Wed, 4 Sep 2024 10:07:05 +0200 Subject: [PATCH 5/8] Added lock functionality --- README.md | 40 +++++ cs3client/file.py | 341 +++++++++++++++++++++++++++++++++++---- examples/lock_example.py | 65 ++++++++ 3 files changed, 418 insertions(+), 28 deletions(-) create mode 100644 examples/lock_example.py diff --git a/README.md b/README.md index 669d715..6f34500 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,46 @@ res = client.file.list_dir(auth.get_token(), list_directory_resource) # readfile file_res = client.file.read_file(auth.get_token(), rename_resource) ``` +### Lock Example +```python + +WEBDAV_LOCK_PREFIX = 'opaquelocktoken:797356a8-0500-4ceb-a8a0-c94c8cde7eba' + + +def encode_lock(lock): + '''Generates the lock payload for the storage given the raw metadata''' + if lock: + return WEBDAV_LOCK_PREFIX + ' ' + b64encode(lock.encode()).decode() + return None + +resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/lock_test.txt") + +# Set lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id=encode_lock("some_lock")) + +# Get lock +res = client.file.get_lock(auth_token, resource) +if res is not None: + lock_id = res["lock_id"] + print(res) + +# Unlock +res = client.file.unlock(auth_token, resource, app_name="a", lock_id=lock_id) + +# Refresh lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id=encode_lock("some_lock")) +res = client.file.refresh_lock( + auth_token, resource, app_name="a", lock_id=encode_lock("new_lock"), existing_lock_id=lock_id +) + +if res is not None: + print(res) + +res = client.file.get_lock(auth_token, resource) +if res is not None: + print(res) + +``` ### Share Example ```python diff --git a/cs3client/file.py b/cs3client/file.py index 43f65e8..da94551 100644 --- a/cs3client/file.py +++ b/cs3client/file.py @@ -15,12 +15,16 @@ import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3sp from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub import cs3.types.v1beta1.types_pb2 as types +import cs3.rpc.v1beta1.code_pb2 as cs3code + from .config import Config from .exceptions.exceptions import AuthenticationException, FileLockedException from .cs3resource import Resource from .statuscodehandler import StatusCodeHandler +LOCK_ATTR_KEY = 'cs3client.advlock' + class File: """ @@ -34,10 +38,10 @@ def __init__( """ Initializes the File class with configuration, logger, auth, gateway stub, and status code handler. - :param config: Config object containing the configuration parameters. - :param log: Logger instance for logging. - :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway. - :param status_code_handler: An instance of the StatusCodeHandler class. + :param config: Config object containing the configuration parameters + :param log: Logger instance for logging + :param gateway: GatewayAPIStub instance for interacting with CS3 Gateway + :param status_code_handler: An instance of the StatusCodeHandler class """ self._config: Config = config self._log: logging.Logger = log @@ -49,7 +53,7 @@ def stat(self, auth_token: tuple, resource: Resource) -> cs3spr.ResourceInfo: Stat a file and return the ResourceInfo object. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: resource to stat. + :param resource: resource to stat :return: cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo (success) :raises: NotFoundException (File not found) :raises: AuthenticationException (Authentication Failed) @@ -66,14 +70,15 @@ def stat(self, auth_token: tuple, resource: Resource) -> cs3spr.ResourceInfo: ) return res.info - def set_xattr(self, auth_token: tuple, resource: Resource, key: str, value: str) -> None: + def set_xattr(self, auth_token: tuple, resource: Resource, key: str, value: str, lock_id: str = None) -> None: """ Set the extended attribute to for a resource. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) :param resource: resource that has the attribute. - :param key: attribute key. - :param value: value to set. + :param key: attribute key + :param value: value to set + :param lock_id: lock id :return: None (Success) :raises: FileLockedException (File is locked) :raises: AuthenticationException (Authentication Failed) @@ -81,60 +86,65 @@ def set_xattr(self, auth_token: tuple, resource: Resource, key: str, value: str) """ md = cs3spr.ArbitraryMetadata() md.metadata.update({key: value}) # pylint: disable=no-member - req = cs3sp.SetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata=md) + req = cs3sp.SetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata=md, lock_id=lock_id) res = self._gateway.SetArbitraryMetadata(request=req, metadata=[auth_token]) # CS3 storages may refuse to set an xattr in case of lock mismatch: this is an overprotection, # as the lock should concern the file's content, not its metadata, however we need to handle that self._status_code_handler.handle_errors(res.status, "set extended attribute", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked setxattr" trace="{res.status.trace}"') - def remove_xattr(self, auth_token: tuple, resource: Resource, key: str) -> None: + def remove_xattr(self, auth_token: tuple, resource: Resource, key: str, lock_id: str = None) -> None: """ Remove the extended attribute . :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: cs3client resource. - :param key: key for attribute to remove. + :param resource: cs3client resource + :param key: key for attribute to remove + :param lock_id: lock id :return: None (Success) :raises: FileLockedException (File is locked) :raises: AuthenticationException (Authentication failed) :raises: UnknownException (Unknown error) """ - req = cs3sp.UnsetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata_keys=[key]) + req = cs3sp.UnsetArbitraryMetadataRequest(ref=resource.ref, arbitrary_metadata_keys=[key], lock_id=lock_id) res = self._gateway.UnsetArbitraryMetadata(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "remove extended attribute", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked UnsetArbitraryMetaData" trace="{res.status.trace}"') - def rename_file(self, auth_token: tuple, resource: Resource, newresource: Resource) -> None: + def rename_file( + self, auth_token: tuple, resource: Resource, newresource: Resource, lock_id: str = None + ) -> None: """ Rename/move resource to new resource. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: Original resource. - :param newresource: New resource. + :param resource: Original resource + :param newresource: New resource + :param lock_id: lock id :return: None (Success) :raises: NotFoundException (Original resource not found) :raises: FileLockException (Resource is locked) :raises: AuthenticationException (Authentication Failed) :raises: UnknownException (Unknown Error) """ - req = cs3sp.MoveRequest(source=resource.ref, destination=newresource.ref) + req = cs3sp.MoveRequest(source=resource.ref, destination=newresource.ref, lock_id=lock_id) res = self._gateway.Move(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "rename file", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked Move" trace="{res.status.trace}"') - def remove_file(self, auth_token: tuple, resource: Resource) -> None: + def remove_file(self, auth_token: tuple, resource: Resource, lock_id: str = None) -> None: """ Remove a resource. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: Resource to remove. + :param resource: Resource to remove + :param lock_id: lock id :return: None (Success) :raises: AuthenticationException (Authentication Failed) :raises: NotFoundException (Resource not found) :raises: UnknownException (Unknown error) """ - req = cs3sp.DeleteRequest(ref=resource.ref) + req = cs3sp.DeleteRequest(ref=resource.ref, lock_id=lock_id) res = self._gateway.Delete(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "remove file", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked Delete" trace="{res.status.trace}"') @@ -144,7 +154,7 @@ def touch_file(self, auth_token: tuple, resource: Resource) -> None: Create a resource. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: Resource to create. + :param resource: Resource to create :return: None (Success) :raises: FileLockedException (File is locked) :raises: AuthenticationException (Authentication Failed) @@ -158,7 +168,9 @@ def touch_file(self, auth_token: tuple, resource: Resource) -> None: self._status_code_handler.handle_errors(res.status, "touch file", resource.get_file_ref_str()) self._log.debug(f'msg="Invoked TouchFile" trace="{res.status.trace}"') - def write_file(self, auth_token: tuple, resource: Resource, content: str | bytes, size: int) -> None: + def write_file( + self, auth_token: tuple, resource: Resource, content: str | bytes, size: int, lock_md: tuple = ('', '') + ) -> None: """ Write a file using the given userid as access token. The entire content is written and any pre-existing file is deleted (or moved to the previous version if supported), @@ -166,15 +178,17 @@ def write_file(self, auth_token: tuple, resource: Resource, content: str | bytes implementation does not support touchfile. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: Resource to write content to. + :param resource: Resource to write content to :param content: content to write :param size: size of content (optional) + :param lock_md: tuple (, ) :return: None (Success) :raises: FileLockedException (File is locked), :raises: AuthenticationException (Authentication failed) :raises: UnknownException (Unknown error) """ + app_name, lock_id = lock_md tstart = time.time() # prepare endpoint if size == -1: @@ -184,6 +198,7 @@ def write_file(self, auth_token: tuple, resource: Resource, content: str | bytes req = cs3sp.InitiateFileUploadRequest( ref=resource.ref, opaque=types.Opaque(map={"Upload-Length": types.OpaqueEntry(decoder="plain", value=str.encode(str(size)))}), + lock_id=lock_id ) res = self._gateway.InitiateFileUpload(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "write file", resource.get_file_ref_str()) @@ -202,12 +217,16 @@ def write_file(self, auth_token: tuple, resource: Resource, content: str | bytes "File-Size": str(size), "X-Reva-Transfer": protocol.token, **dict([auth_token]), + "X-Lock-Id": lock_id, + "X-Lock_Holder": app_name, } else: headers = { "Upload-Length": str(size), "X-Reva-Transfer": protocol.token, **dict([auth_token]), + "X-Lock-Id": lock_id, + "X-Lock_Holder": app_name, } putres = requests.put( url=protocol.upload_endpoint, @@ -249,12 +268,13 @@ def write_file(self, auth_token: tuple, resource: Resource, content: str | bytes f'elapsedTimems="{(tend - tstart) * 1000:.1f}"' ) - def read_file(self, auth_token: tuple, resource: Resource) -> Generator[bytes, None, None]: + def read_file(self, auth_token: tuple, resource: Resource, lock_id: str = None) -> Generator[bytes, None, None]: """ Read a file. Note that the function is a generator, managed by the app server. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: Resource to read. + :param resource: Resource to read + :param lock_id: lock id :return: Generator[Bytes, None, None] (Success) :raises: NotFoundException (Resource not found) :raises: AuthenticationException (Authentication Failed) @@ -263,7 +283,7 @@ def read_file(self, auth_token: tuple, resource: Resource) -> Generator[bytes, N tstart = time.time() # prepare endpoint - req = cs3sp.InitiateFileDownloadRequest(ref=resource.ref) + req = cs3sp.InitiateFileDownloadRequest(ref=resource.ref, lock_id=lock_id) res = self._gateway.InitiateFileDownload(request=req, metadata=[auth_token]) self._status_code_handler.handle_errors(res.status, "read file", resource.get_file_ref_str()) tend = time.time() @@ -304,7 +324,7 @@ def make_dir(self, auth_token: tuple, resource: Resource) -> None: Create a directory. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: Direcotry to create. + :param resource: Direcotry to create :return: None (Success) :raises: FileLockedException (File is locked) :raises: AuthenticationException (Authentication failed) @@ -322,7 +342,7 @@ def list_dir( List the contents of a directory, note that the function is a generator. :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) - :param resource: the directory. + :param resource: the directory :return: Generator[cs3.storage.provider.v1beta1.resources_pb2.ResourceInfo, None, None] (Success) :raises: NotFoundException (Resrouce not found) :raises: AuthenticationException (Authentication Failed) @@ -334,3 +354,268 @@ def list_dir( self._log.debug(f'msg="Invoked ListContainer" trace="{res.status.trace}"') for info in res.infos: yield info + + def _set_lock_using_xattr(self, auth_token, resource: Resource, app_name: str, lock_id: int | str) -> None: + """" + Set a lock to a resource with the given value metadata and appname as holder + + :param resource: Resource to set lock to + :param app_name: Application name + :param lock_id: Metadata lock value + :return: None (Success) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute SetLock" {resource.get_file_ref_str()}" value="{lock_id}"') + try: + # The stat can raise a KeyError if the metadata (lock) attribute has not been set yet + # e.g. the file is not locked + _ = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + except KeyError: + expiration = int(time.time() + self._config.lock_expiration) + self.set_xattr(auth_token, resource, LOCK_ATTR_KEY, f"{app_name}!{lock_id}!{expiration}", None) + return + + def set_lock(self, auth_token: tuple, resource: Resource, app_name: str, lock_id: int | str) -> None: + """ + Set a lock to a resource with the given value and appname as holder + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to set lock to + :param app_name: Application name + :param lock_id: encoded lock_id or metadata lock value if using xattr + :return: None (Success) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + self._set_lock_using_xattr(auth_token, resource, app_name, lock_id) + return + + lock = cs3spr.Lock( + type=cs3spr.LOCK_TYPE_WRITE, + lock_id=lock_id, + app_name=app_name, + expiration={"seconds": int(time.time() + + self._config.lock_expiration)}, + ) + req = cs3sp.SetLockRequest(ref=resource.ref, lock=lock) + res = self._gateway.SetLock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, set the lock using xattr + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + self._set_lock_using_xattr(auth_token, resource, app_name, lock_id) + return + + self._status_code_handler.handle_errors(res.status, "set lock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked SetLock" {resource.get_file_ref_str()}" ' + f'value="{lock_id} result="{res.status.trace}"') + + def _get_lock_using_xattr(self, auth_token: tuple, resource: Resource) -> dict: + """ + Get the lock metadata for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to get lock from + :return: dictionary (KEYS: lock_id, type, app_name, user, expiration) (Success) + :return: None (No lock set) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute getlock" {resource.get_file_ref_str()}"') + try: + currvalue = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + values = currvalue.split("!") + return { + "lock_id": values[1], + "type": 2, # LOCK_TYPE_WRITE, though this is advisory! + "app_name": values[0], + "user": {}, + "expiration": int(values[2]), + } + except KeyError: + return None + + def get_lock(self, auth_token: tuple, resource: Resource) -> cs3spr.Lock | dict | None: + """ + Get the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to get lock from + :return: dictionary (KEYS: lock_id, type, app_name, user, expiration) (Success) + :return: None (No lock set) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + return self._get_lock_using_xattr(auth_token, resource) + + req = cs3sp.GetLockRequest(ref=resource.ref) + res = self._gateway.GetLock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, get the lock using xattr + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + return self._get_lock_using_xattr(auth_token, resource) + + self._status_code_handler.handle_errors(res.status, "get lock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked GetLock" {resource.get_file_ref_str()}" result="{res.status.trace}"') + + # rebuild a dict corresponding to the internal JSON structure used by Reva + return { + "lock_id": res.lock.lock_id, + "type": res.lock.type, + "app_name": res.lock.app_name, + "user": ( + {"opaque_id": res.lock.user.opaque_id, "idp": res.lock.user.idp, "type": res.lock.user.type} + if res.lock.user.opaque_id + else {} + ), + "expiration": {"seconds": res.lock.expiration.seconds}, + } + + def _refresh_lock_using_xattr( + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: str | int, existing_lock_id: str | int = None + ) -> None: + """ + Refresh the lock metadata for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to refresh lock for + :param app_name: Application name + :param lock_id: metadata value to set + :param existing_lock_id: existing metadata vlue + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute RefreshLock" {resource.get_file_ref_str()}" value="{lock_id}"') + try: + # The stat can raise a KeyError if the metadata (lock) attribute has not been set yet + # e.g. the file is not locked + currvalue = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + values = currvalue.split("!") + if values[0] == app_name and (not existing_lock_id or values[1] == existing_lock_id): + raise KeyError + self._log.info( + f'Failed precondition on RefreshLock" {resource.get_file_ref_str()}" appname="{app_name}" ' + f'value="{lock_id} previouslock="{currvalue}"' + ) + raise FileLockedException() + except KeyError: + expiration = int(time.time() + self._config.lock_expiration) + self.set_xattr(auth_token, resource, LOCK_ATTR_KEY, f"{app_name}!{lock_id}!{expiration}", None) + return + + def refresh_lock( + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: str | int, + existing_lock_id: str | int = None + ): + """ + Refresh the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to refresh lock for + :param app_name: Application name + :param lock_id: encoded lock_id or metadata lock value if using xattr + :param existing_lock_id: encoded lock_id or metadata lock value if using xattr + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + self._refresh_lock_using_xattr(auth_token, resource, app_name, lock_id, existing_lock_id) + return + + lock = cs3spr.Lock( + type=cs3spr.LOCK_TYPE_WRITE, + app_name=app_name, + lock_id=lock_id, + expiration={"seconds": int(time.time() + self._config.lock_expiration)}, + ) + req = cs3sp.RefreshLockRequest(ref=resource.ref, lock=lock, existing_lock_id=existing_lock_id) + res = self._gateway.RefreshLock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, refresh the lock using xattr + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + self._refresh_lock_using_xattr(auth_token, resource, app_name, lock_id, existing_lock_id) + return + self._status_code_handler.handle_errors(res.status, "refresh lock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked RefreshLock" {resource.get_file_ref_str()} result="{res.status.trace}" ' + f'value="{lock_id}" old_value="{existing_lock_id}"') + + def _unlock_using_xattr(self, auth_token: tuple, resource: Resource, app_name: str, lock_id: str | int) -> None: + """ + Remove the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to unlock + :param app_name: Application name + :param lock_id: metadata lock value + :return: None (Success) + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + self._log.debug(f'msg="Using xattrs to execute unlock" {resource.get_file_ref_str()}" value="{lock_id}"') + try: + # The stat can raise a KeyError if the metadata (lock) attribute has not been set yet + # e.g. the file is not locked + currvalue = self.stat(auth_token, resource).arbitrary_metadata.metadata[LOCK_ATTR_KEY] + values = currvalue.split("!") + if values[0] == app_name and values[1] == lock_id: + raise KeyError + self._log.info( + f'Failed precondition on unlock" {resource.get_file_ref_str()}" appname="{app_name}" ' + f'value={lock_id} previouslock="{currvalue}"' + ) + raise FileLockedException() + except KeyError: + self.remove_xattr(auth_token, resource, LOCK_ATTR_KEY, None) + return + + def unlock(self, auth_token: tuple, resource: Resource, app_name, lock_id: str | int): + """ + Remove the lock for the given filepath + + :param auth_token: tuple in the form ('x-access-token', ) (see auth.get_token/auth.check_token) + :param resource: Resource to unlock + :param app_name: app_name + :param lock_id: encoded lock_id or metadata lock value if using xattr + :return: None + :raises: FileLockedException (File is locked) + :raises: NotFoundException (File not found) + :raises: AuthenticationException (Authentication Failed) + :raises: UnknownException (Unknown Error) + """ + # fallback to xattr if the storage does not support locks + if self._config.lock_by_setting_attr and self._config.lock_not_impl: + self._unlock_using_xattr(auth_token, resource, app_name, lock_id) + return + + lock = cs3spr.Lock(type=cs3spr.LOCK_TYPE_WRITE, app_name=app_name, lock_id=lock_id) + req = cs3sp.UnlockRequest(ref=resource.ref, lock=lock) + res = self._gateway.Unlock(request=req, metadata=[auth_token]) + + # if the storage does not support locks, set the lock using xattr and retry + if res.status.code == cs3code.CODE_UNIMPLEMENTED and self._config.lock_by_setting_attr: + self._config.lock_not_impl = True + self._unlock_using_xattr(auth_token, resource, app_name, lock_id) + return + + self._status_code_handler.handle_errors(res.status, "unlock", resource.get_file_ref_str()) + self._log.debug(f'msg="Invoked Unlock" {resource.get_file_ref_str()} result="{res.status.trace}" ' + f'value="{lock_id}"') diff --git a/examples/lock_example.py b/examples/lock_example.py new file mode 100644 index 0000000..ce86201 --- /dev/null +++ b/examples/lock_example.py @@ -0,0 +1,65 @@ +""" +lock_example.py + +Example script to demonstrate the usage of the app API in the CS3Client class. +note that these are examples, and is not meant to be run as a script. + +Authors: Rasmus Welander, Diogo Castro, Giuseppe Lo Presti. +Emails: rasmus.oscar.welander@cern.ch, diogo.castro@cern.ch, giuseppe.lopresti@cern.ch +Last updated: 30/08/2024 +""" + +import logging +import configparser +from cs3client.cs3client import CS3Client +from cs3client.auth import Auth +from cs3client.cs3resource import Resource + + +config = configparser.ConfigParser() +with open("default.conf") as fdef: + config.read_file(fdef) +log = logging.getLogger(__name__) + +client = CS3Client(config, "cs3client", log) +auth = Auth(client) +# Set the client id (can also be set in the config) +auth.set_client_id("") +# Set client secret (can also be set in config) +auth.set_client_secret("") +# Checks if token is expired if not return ('x-access-token', ) +# if expired, request a new token from reva +auth_token = auth.get_token() + +# OR if you already have a reva token +# Checks if token is expired if not return (x-access-token', ) +# if expired, throws an AuthenticationException (so you can refresh your reva token) +token = "" +auth_token = Auth.check_token(token) + +resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/lock_test.txt") + +# Set lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id="some_lock") + +# Get lock +res = client.file.get_lock(auth_token, resource) +if res is not None: + lock_id = res["lock_id"] + print(res) + +# Unlock +res = client.file.unlock(auth_token, resource, app_name="a", lock_id=lock_id) + +# Refresh lock +client.file.set_lock(auth_token, resource, app_name="a", lock_id="some_lock") +res = client.file.refresh_lock( + auth_token, resource, app_name="a", lock_id="new_lock", existing_lock_id=lock_id +) + +if res is not None: + print(res) + +res = client.file.get_lock(auth_token, resource) +if res is not None: + print(res) From 455b53fb8dcad0cd888fa6cebe93826bcaab678f Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Wed, 4 Sep 2024 16:43:56 +0200 Subject: [PATCH 6/8] [CI] use Python 3.9 as that is what we have in Alma9 --- .github/workflows/ci-tests.yml | 4 ++-- cs3client/auth.py | 7 +++--- cs3client/cs3resource.py | 27 +++++++++++---------- cs3client/file.py | 27 +++++++++++++-------- cs3client/statuscodehandler.py | 44 ++++++++++++++++++++-------------- 5 files changed, 63 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 456d92e..56d2173 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -15,10 +15,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python 3.12 + - name: Set up Python 3.9 uses: actions/setup-python@v3 with: - python-version: "3.12" + python-version: "3.9" - name: Install dependencies run: | diff --git a/cs3client/auth.py b/cs3client/auth.py index 56b4f89..a594094 100644 --- a/cs3client/auth.py +++ b/cs3client/auth.py @@ -10,6 +10,7 @@ import jwt import datetime import logging +from typing import Union from cs3.gateway.v1beta1.gateway_api_pb2 import AuthenticateRequest from cs3.auth.registry.v1beta1.registry_api_pb2 import ListAuthProvidersRequest from cs3.gateway.v1beta1.gateway_api_pb2_grpc import GatewayAPIStub @@ -38,9 +39,9 @@ def __init__(self, cs3_client: CS3Client) -> None: self._log: logging.Logger = cs3_client._log self._config: Config = cs3_client._config # The user should be able to change the client secret (e.g. token) and client id at runtime - self._client_secret: str | None = self._config.auth_client_secret - self._client_id: str | None = self._config.auth_client_id - self._token: str | None = None + self._client_secret: Union[str, None] = self._config.auth_client_secret + self._client_id: Union[str, None] = self._config.auth_client_id + self._token: Union[str, None] = None def set_client_secret(self, token: str) -> None: """ diff --git a/cs3client/cs3resource.py b/cs3client/cs3resource.py index 6a19ee3..91097b5 100644 --- a/cs3client/cs3resource.py +++ b/cs3client/cs3resource.py @@ -7,6 +7,7 @@ """ import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr +from typing import Union class Resource: @@ -26,12 +27,12 @@ class Resource: def __init__( self, - abs_path: str | None = None, - rel_path: str | None = None, - opaque_id: str | None = None, - parent_id: str | None = None, - storage_id: str | None = None, - space_id: str | None = None, + abs_path: Union[str, None] = None, + rel_path: Union[str, None] = None, + opaque_id: Union[str, None] = None, + parent_id: Union[str, None] = None, + storage_id: Union[str, None] = None, + space_id: Union[str, None] = None, ) -> None: """ initializes the Resource class, either abs_path, rel_path or opaque_id is required @@ -45,15 +46,15 @@ def __init__( :param storage_id: storage id (optional) :param space_id: space id (optional) """ - self._abs_path: str | None = abs_path - self._rel_path: str | None = rel_path - self._parent_id: str | None = parent_id - self._opaque_id: str | None = opaque_id - self._space_id: str | None = space_id - self._storage_id: str | None = storage_id + self._abs_path: Union[str, None] = abs_path + self._rel_path: Union[str, None] = rel_path + self._parent_id: Union[str, None] = parent_id + self._opaque_id: Union[str, None] = opaque_id + self._space_id: Union[str, None] = space_id + self._storage_id: Union[str, None] = storage_id @classmethod - def from_file_ref_and_endpoint(cls, file: str, endpoint: str | None = None) -> "Resource": + def from_file_ref_and_endpoint(cls, file: str, endpoint: Union[str, None] = None) -> "Resource": """ Extracts the attributes from the file and endpoint and returns a resource. diff --git a/cs3client/file.py b/cs3client/file.py index da94551..ee08b82 100644 --- a/cs3client/file.py +++ b/cs3client/file.py @@ -10,6 +10,7 @@ import logging import http import requests +from typing import Union from typing import Generator import cs3.storage.provider.v1beta1.resources_pb2 as cs3spr import cs3.storage.provider.v1beta1.provider_api_pb2 as cs3sp @@ -169,7 +170,8 @@ def touch_file(self, auth_token: tuple, resource: Resource) -> None: self._log.debug(f'msg="Invoked TouchFile" trace="{res.status.trace}"') def write_file( - self, auth_token: tuple, resource: Resource, content: str | bytes, size: int, lock_md: tuple = ('', '') + self, auth_token: tuple, resource: Resource, content: Union[str, bytes], size: int, + lock_md: tuple = ('', '') ) -> None: """ Write a file using the given userid as access token. The entire content is written @@ -307,9 +309,11 @@ def read_file(self, auth_token: tuple, resource: Resource, lock_id: str = None) raise IOError(e) data = fileget.iter_content(self._config.chunk_size) if fileget.status_code != http.client.OK: + # status.message.replace('"', "'") is not allowed inside f strings python<3.12 + status_msg = fileget.reason.replace('"', "'") self._log.error( f'msg="Error downloading file from Reva" code="{fileget.status_code}" ' - f'reason="{fileget.reason.replace('"', "'")}"' + f'reason="{status_msg}"' ) raise IOError(fileget.reason) else: @@ -355,7 +359,7 @@ def list_dir( for info in res.infos: yield info - def _set_lock_using_xattr(self, auth_token, resource: Resource, app_name: str, lock_id: int | str) -> None: + def _set_lock_using_xattr(self, auth_token, resource: Resource, app_name: str, lock_id: Union[int, str]) -> None: """" Set a lock to a resource with the given value metadata and appname as holder @@ -377,7 +381,7 @@ def _set_lock_using_xattr(self, auth_token, resource: Resource, app_name: str, l self.set_xattr(auth_token, resource, LOCK_ATTR_KEY, f"{app_name}!{lock_id}!{expiration}", None) return - def set_lock(self, auth_token: tuple, resource: Resource, app_name: str, lock_id: int | str) -> None: + def set_lock(self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[int, str]) -> None: """ Set a lock to a resource with the given value and appname as holder @@ -440,7 +444,7 @@ def _get_lock_using_xattr(self, auth_token: tuple, resource: Resource) -> dict: except KeyError: return None - def get_lock(self, auth_token: tuple, resource: Resource) -> cs3spr.Lock | dict | None: + def get_lock(self, auth_token: tuple, resource: Resource) -> Union[cs3spr.Lock, dict, None]: """ Get the lock for the given filepath @@ -481,7 +485,8 @@ def get_lock(self, auth_token: tuple, resource: Resource) -> cs3spr.Lock | dict } def _refresh_lock_using_xattr( - self, auth_token: tuple, resource: Resource, app_name: str, lock_id: str | int, existing_lock_id: str | int = None + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[str, int], + existing_lock_id: Union[str, int] = None ) -> None: """ Refresh the lock metadata for the given filepath @@ -516,8 +521,8 @@ def _refresh_lock_using_xattr( return def refresh_lock( - self, auth_token: tuple, resource: Resource, app_name: str, lock_id: str | int, - existing_lock_id: str | int = None + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[str, int], + existing_lock_id: Union[str, int] = None ): """ Refresh the lock for the given filepath @@ -556,7 +561,9 @@ def refresh_lock( self._log.debug(f'msg="Invoked RefreshLock" {resource.get_file_ref_str()} result="{res.status.trace}" ' f'value="{lock_id}" old_value="{existing_lock_id}"') - def _unlock_using_xattr(self, auth_token: tuple, resource: Resource, app_name: str, lock_id: str | int) -> None: + def _unlock_using_xattr( + self, auth_token: tuple, resource: Resource, app_name: str, lock_id: Union[str, int] + ) -> None: """ Remove the lock for the given filepath @@ -587,7 +594,7 @@ def _unlock_using_xattr(self, auth_token: tuple, resource: Resource, app_name: s self.remove_xattr(auth_token, resource, LOCK_ATTR_KEY, None) return - def unlock(self, auth_token: tuple, resource: Resource, app_name, lock_id: str | int): + def unlock(self, auth_token: tuple, resource: Resource, app_name, lock_id: Union[str, int]): """ Remove the lock for the given filepath diff --git a/cs3client/statuscodehandler.py b/cs3client/statuscodehandler.py index bed57f5..c468212 100644 --- a/cs3client/statuscodehandler.py +++ b/cs3client/statuscodehandler.py @@ -20,57 +20,65 @@ def __init__(self, log: logging.Logger, config: Config) -> None: self._log = log self._config = config - def _log_not_found_info(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_not_found_info(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.info( f'msg="Not found on {operation}" {msg + " " if msg else ""} ' f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' - f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_authentication_error(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_authentication_error( + self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None + ) -> None: self._log.error( f'msg="Authentication failed on {operation}" {msg + " " if msg else ""}' f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' - f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_unknown_error(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_unknown_error(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.error( f'msg="Failed to {operation}, unknown error" {msg + " " if msg else ""}' f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' - f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_precondition_info(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_precondition_info( + self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None + ) -> None: self._log.info( f'msg="Failed precondition on {operation}" {msg + " " if msg else ""}' f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' - f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_already_exists(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_already_exists(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.info( f'msg="Already exists on {operation}" {msg + " " if msg else ""}' f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' - f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' + f'trace="{status.trace}" reason="{status_msg}"' ) - def _log_unimplemented(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + def _log_unimplemented(self, status: cs3status.Status, operation: str, status_msg: str, msg: str = None) -> None: self._log.info( f'msg="Invoked {operation} on unimplemented feature" {msg + " " if msg else ""}' f'userid="{self._config.auth_client_id if self._config.auth_client_id else "no_id_set"}" ' - f'trace="{status.trace}" reason="{status.message.replace('"', "'")}"' + f'trace="{status.trace}" reason="{status_msg}"' ) def handle_errors(self, status: cs3status.Status, operation: str, msg: str = None) -> None: + if status.code == cs3code.CODE_OK: return + # status.message.replace('"', "'") is not allowed inside f strings python<3.12 + status_message = status.message.replace('"', "'") + if status.code == cs3code.CODE_FAILED_PRECONDITION or status.code == cs3code.CODE_ABORTED: - self._log_precondition_info(status, operation, msg) + self._log_precondition_info(status, operation, status_message, msg) raise FileLockedException(f'Failed precondition: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_ALREADY_EXISTS: - self._log_already_exists(status, operation, msg) + self._log_already_exists(status, operation, status_message, msg) raise AlreadyExistsException(f'Resource already exists: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_UNIMPLEMENTED: @@ -78,11 +86,11 @@ def handle_errors(self, status: cs3status.Status, operation: str, msg: str = Non raise UnimplementedException(f'Unimplemented feature: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_NOT_FOUND: - self._log_not_found_info(status, operation, msg) + self._log_not_found_info(status, operation, status_message, msg) raise NotFoundException(f'Not found: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code == cs3code.CODE_UNAUTHENTICATED: - self._log_authentication_error(status, operation, msg) + self._log_authentication_error(status, operation, status_message, msg) raise AuthenticationException(f'Operation not permitted: operation="{operation}" ' f'status_code="{status.code}" message="{status.message}"') if status.code != cs3code.CODE_OK: @@ -90,8 +98,8 @@ def handle_errors(self, status: cs3status.Status, operation: str, msg: str = Non self._log.info(f'msg="Invoked {operation} on missing file" ') raise NotFoundException( message=f'No such file or directory: operation="{operation}" ' - f'status_code="{status.code}" message="{status.message}"' + f'status_code="{status.code}" message="{status.message}"' ) - self._log_unknown_error(status, operation, msg) + self._log_unknown_error(status, operation, status_message, msg) raise UnknownException(f'Unknown Error: operation="{operation}" status_code="{status.code}" ' f'message="{status.message}"') From a2e65b892121471d3d28faf429cf6d949dae5645 Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Fri, 6 Sep 2024 10:32:54 +0200 Subject: [PATCH 7/8] Updated .gitignore to include MacOS files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 34de50d..4762aba 100644 --- a/.gitignore +++ b/.gitignore @@ -157,6 +157,9 @@ dmypy.json # Cython debug symbols cython_debug/ +# MacOS +.DS_Store + # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore From 8c016f104c2d363e97f7b32dd8964f2f243e23cf Mon Sep 17 00:00:00 2001 From: Rasmus Oscar Welander Date: Fri, 6 Sep 2024 10:41:11 +0200 Subject: [PATCH 8/8] Adapted examples to create a resource instance using abs_path --- README.md | 18 +++++++++--------- examples/app_api_example.py | 2 +- examples/checkpoints_api_example.py | 2 +- examples/file_api_example.py | 12 ++++++------ examples/lock_example.py | 2 +- examples/shares_api_example.py | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6f34500..dfd58fb 100644 --- a/README.md +++ b/README.md @@ -146,15 +146,15 @@ auth_token = Auth.check_token(token) ### File Example ```python # mkdir -directory_resource = Resource.from_file_ref_and_endpoint(f"/eos/user/r/rwelande/test_directory") +directory_resource = Resource(abs_path=f"/eos/user/r/rwelande/test_directory") res = client.file.make_dir(auth.get_token(), directory_resource) # touchfile -touch_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/touch_file.txt") +touch_resource = Resource(abs_path="/eos/user/r/rwelande/touch_file.txt") res = client.file.touch_file(auth.get_token(), touch_resource) # setxattr -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") +resource = Resource(abs_path="/eos/user/r/rwelande/text_file.txt") res = client.file.set_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime", str(1720696124)) # rmxattr @@ -167,7 +167,7 @@ res = client.file.stat(auth.get_token(), resource) res = client.file.remove_file(auth.get_token(), touch_resource) # rename -rename_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/rename_file.txt") +rename_resource = Resource(abs_path="/eos/user/r/rwelande/rename_file.txt") res = client.file.rename_file(auth.get_token(), resource, rename_resource) # writefile @@ -176,7 +176,7 @@ size = len(content) res = client.file.write_file(auth.get_token(), rename_resource, content, size) # listdir -list_directory_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande") +list_directory_resource = Resource(abs_path="/eos/user/r/rwelande") res = client.file.list_dir(auth.get_token(), list_directory_resource) @@ -195,7 +195,7 @@ def encode_lock(lock): return WEBDAV_LOCK_PREFIX + ' ' + b64encode(lock.encode()).decode() return None -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/lock_test.txt") +resource = Resource(abs_path="/eos/user/r/rwelande/lock_test.txt") # Set lock client.file.set_lock(auth_token, resource, app_name="a", lock_id=encode_lock("some_lock")) @@ -227,7 +227,7 @@ if res is not None: ### Share Example ```python # Create share # -resource = Resource.from_file_ref_and_endpoint("/eos/user/r//text.txt") +resource = Resource(abs_path="/eos/user/r//text.txt") resource_info = client.file.stat(auth.get_token(), resource) user = client.user.get_user_by_claim("username", "") res = client.share.create_share(auth.get_token(), resource_info, user.id.opaque_id, user.id.idp, "EDITOR", "USER") @@ -308,14 +308,14 @@ res = client.user.get_user_by_claim("username", "rwelande") res = client.app.list_app_providers(auth.get_token()) # open_in_app -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/collabora.odt") +resource = Resource(abs_path="/eos/user/r/rwelande/collabora.odt") res = client.app.open_in_app(auth.get_token(), resource) ``` ### Checkpoint Example ```python # list file versions -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/test.md") +resource = Resource(abs_path="/eos/user/r/rwelande/test.md") res = client.checkpoint.list_file_versions(auth.get_token(), resource) # restore file version diff --git a/examples/app_api_example.py b/examples/app_api_example.py index 608de6c..21999ff 100644 --- a/examples/app_api_example.py +++ b/examples/app_api_example.py @@ -42,7 +42,7 @@ print(res) # open_in_app -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/collabora.odt") +resource = Resource(abs_path="/eos/user/r/rwelande/collabora.odt") res = client.app.open_in_app(auth.get_token(), resource) if res is not None: print(res) diff --git a/examples/checkpoints_api_example.py b/examples/checkpoints_api_example.py index 7e2b9de..5fa03d8 100644 --- a/examples/checkpoints_api_example.py +++ b/examples/checkpoints_api_example.py @@ -38,7 +38,7 @@ res = None -markdown_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/test.md") +markdown_resource = Resource(abs_path="/eos/user/r/rwelande/test.md") res = client.checkpoint.list_file_versions(auth.get_token(), markdown_resource) diff --git a/examples/file_api_example.py b/examples/file_api_example.py index db5fcdc..fb25bed 100644 --- a/examples/file_api_example.py +++ b/examples/file_api_example.py @@ -47,14 +47,14 @@ # mkdir for i in range(1, 4): - directory_resource = Resource.from_file_ref_and_endpoint(f"/eos/user/r/rwelande/test_directory{i}") + directory_resource = Resource(abs_path=f"/eos/user/r/rwelande/test_directory{i}") res = client.file.make_dir(auth.get_token(), directory_resource) if res is not None: print(res) # touchfile -touch_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/touch_file.txt") -text_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") +touch_resource = Resource(abs_path="/eos/user/r/rwelande/touch_file.txt") +text_resource = Resource(abs_path="/eos/user/r/rwelande/text_file.txt") res = client.file.touch_file(auth.get_token(), touch_resource) res = client.file.touch_file(auth.get_token(), text_resource) @@ -62,7 +62,7 @@ print(res) # setxattr -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text_file.txt") +resource = Resource(abs_path="/eos/user/r/rwelande/text_file.txt") res = client.file.set_xattr(auth.get_token(), resource, "iop.wopi.lastwritetime", str(1720696124)) if res is not None: @@ -89,7 +89,7 @@ res = client.file.touch_file(auth.get_token(), touch_resource) # rename -rename_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/rename_file.txt") +rename_resource = Resource(abs_path="/eos/user/r/rwelande/rename_file.txt") res = client.file.rename_file(auth.get_token(), resource, rename_resource) if res is not None: @@ -110,7 +110,7 @@ print(res) # listdir -list_directory_resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande") +list_directory_resource = Resource(abs_path="/eos/user/r/rwelande") res = client.file.list_dir(auth.get_token(), list_directory_resource) first_item = next(res, None) diff --git a/examples/lock_example.py b/examples/lock_example.py index ce86201..6438af2 100644 --- a/examples/lock_example.py +++ b/examples/lock_example.py @@ -37,7 +37,7 @@ token = "" auth_token = Auth.check_token(token) -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/lock_test.txt") +resource = Resource(abs_path="/eos/user/r/rwelande/lock_test.txt") # Set lock client.file.set_lock(auth_token, resource, app_name="a", lock_id="some_lock") diff --git a/examples/shares_api_example.py b/examples/shares_api_example.py index d3dd4f8..f78a1fe 100644 --- a/examples/shares_api_example.py +++ b/examples/shares_api_example.py @@ -39,7 +39,7 @@ res = None # Create share # -resource = Resource.from_file_ref_and_endpoint("/eos/user/r/rwelande/text.txt") +resource = Resource(abs_path="/eos/user/r/rwelande/text.txt") resource_info = client.file.stat(auth.get_token(), resource) # VIEWER