Merge pull request #23 from IanMeyers/main

Version 1.0.17

Author: IanMeyers

setup.py

@@ -3,10 +3,10 @@
 from setuptools import find_packages, setup
 
 setup(
-    install_requires=['boto3~=1.26.46',
+    install_requires=['boto3~=1.26.121',
                       'pystache~=0.6.0',
-                      'setuptools~=65.6.3',
+                      'setuptools~=67.7.2',
                       'shortuuid~=1.0.11',
-                      'botocore~=1.29.46'],
+                      'botocore~=1.29.121'],
     include_package_data=True
 )

src/cli-examples.txt

@@ -27,7 +27,7 @@ export DATA_MESH_ACCOUNT=<my account id>
 ./data-mesh-cli deny-subscription --credentials_file $CREDS_FILE --request-id 4GaXSebUtSvzXQXv7mZt3M --decision_notes "No Way Dude!"
 
 # change your mind and approve it now
-./data-mesh-cli approve-subscription --data_mesh_account_id $DATA_MESH_ACCOUNT  --region_name $AWS_REGION --use_credentials "{\"AccessKeyId\":\"my access key\",\"SecretAccessKey\":\"my secret key\"}" --request-id <request> --grant-permissions <permissions allowed to the consumer> --grantable-permissions <permissions the consumer can pass on. remove parameter for no grantable permissions> --decision_notes "Notes attached to the approval"
+./data-mesh-cli approve-subscription --data_mesh_account_id $DATA_MESH_ACCOUNT --region_name $AWS_REGION --use_credentials "{\"AccessKeyId\":\"my access key\",\"SecretAccessKey\":\"my secret key\"}" --request-id <request> --grant-permissions <permissions allowed to the consumer> --grantable-permissions <permissions the consumer can pass on. remove parameter for no grantable permissions> --decision_notes "Notes attached to the approval"
 ./data-mesh-cli approve-subscription --credentials_file $CREDS_FILE --request-id 4GaXSebUtSvzXQXv7mZt3M --grant-permissions SELECT --decision_notes "Changed my mind"
 
 # modify the subscription to allow an additional grant

src/data_mesh_util/DataMeshAdmin.py

@@ -63,6 +63,9 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo
 
         self._logger.debug(f"Running as {self._current_identity.get('Arn')}")
 
+        if self._log_level == 'DEBUG':
+            utils.log_instance_signature(self, self._logger)
+
     def _create_template_config(self, config: dict):
         if config is None:
             config = {}

src/data_mesh_util/DataMeshConsumer.py

@@ -87,6 +87,9 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo
         self._mesh_automator = ApiAutomator(target_account=self._data_mesh_account_id,
                                             session=self._ro_session, log_level=self._log_level)
 
+        if self._log_level == 'DEBUG':
+            utils.log_instance_signature(self, self._logger)
+
     def request_access_to_product(self, owner_account_id: str, database_name: str,
                                   tables: list, request_permissions: list) -> dict:
         '''

src/data_mesh_util/DataMeshMacros.py

@@ -1,5 +1,6 @@
 from data_mesh_util.lib.constants import *
 from data_mesh_util import DataMeshAdmin as data_mesh_admin
+import data_mesh_util.lib.utils as utils
 
 
 class DataMeshMacros:
@@ -13,6 +14,9 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo
         self._region = region_name
         self._log_level = log_level
 
+        if self._log_level == 'DEBUG':
+            utils.log_instance_signature(self, self._logger)
+
     def bootstrap_account(self, account_type: str, mesh_credentials, account_credentials, crawler_role_arn: str = None):
         # create a data mesh admin for the mesh account
         mesh_admin = data_mesh_admin.DataMeshAdmin(

src/data_mesh_util/DataMeshProducer.py

@@ -14,7 +14,6 @@
 class DataMeshProducer:
     _data_mesh_account_id = None
     _data_producer_account_id = None
-    _data_mesh_manager_role_arn = None
     _session = None
     _iam_client = None
     _sts_client = None
@@ -58,13 +57,16 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo
 
         self._data_producer_identity = self._sts_client.get_caller_identity()
         self._data_producer_account_id = self._data_producer_identity.get('Account')
+        producer_role_name = utils.get_central_role_name(self._data_producer_account_id, PRODUCER)
+        self._data_producer_role_arn = utils.get_role_arn(account_id=self._data_mesh_account_id,
+                                                          role_name=producer_role_name)
 
         self._producer_automator = ApiAutomator(target_account=self._data_producer_account_id,
                                                 session=self._session, log_level=self._log_level)
 
         # now assume the DataMeshProducer-<account-id> Role in the Mesh Account
         self._data_mesh_session, self._data_mesh_credentials, self._data_mesh_arn = utils.assume_iam_role(
-            role_name=utils.get_central_role_name(self._data_producer_account_id, PRODUCER),
+            role_name=producer_role_name,
             region_name=self._current_region,
             use_credentials=_producer_credentials,
             target_account=self._data_mesh_account_id
@@ -85,11 +87,15 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo
                                                        region_name=self._current_region,
                                                        log_level=log_level)
 
+        if self._log_level == 'DEBUG':
+            utils.log_instance_signature(self, self._logger)
+
     def _create_mesh_table(self, table_def: dict, data_mesh_glue_client, source_database_name: str,
                            data_mesh_database_name: str,
                            producer_account_id: str,
                            data_mesh_account_id: str, create_public_metadata: bool = True,
-                           expose_table_references_with_suffix: str = "_link", use_original_table_name: bool = False):
+                           expose_table_references_with_suffix: str = "_link",
+                           use_original_table_name: bool = False) -> tuple:
         '''
         API to create a table as a data product in the data mesh
         :param table_def:
@@ -106,7 +112,7 @@ def _create_mesh_table(self, table_def: dict, data_mesh_glue_client, source_data
         # remove properties from a TableInfo object returned from get_table to be compatible with put_table
         keys = [
             'DatabaseName', 'CreateTime', 'UpdateTime', 'CreatedBy', 'IsRegisteredWithLakeFormation', 'CatalogId',
-            'Tags'
+            'Tags', 'VersionId'
         ]
         t = utils.remove_dict_keys(input_dict=table_def, remove_keys=keys)
         t['Owner'] = producer_account_id
@@ -138,8 +144,8 @@ def _create_mesh_table(self, table_def: dict, data_mesh_glue_client, source_data
                 partition_input_list=table_partitions
             )
 
-        # grant access to the producer account
-        perms = ['INSERT', 'SELECT', 'ALTER', 'DELETE', 'DESCRIBE']
+        # grant full access to the producer account
+        perms = ['INSERT', 'SELECT', 'ALTER', 'DELETE', 'DESCRIBE', 'DROP']
         permissions_granted = self._mesh_automator.lf_grant_permissions(
             data_mesh_account_id=self._data_mesh_account_id,
             principal=producer_account_id,
@@ -153,37 +159,37 @@ def _create_mesh_table(self, table_def: dict, data_mesh_glue_client, source_data
         if create_public_metadata is True:
             self._mesh_automator.lf_grant_permissions(
                 data_mesh_account_id=self._data_mesh_account_id,
-                principal=utils.get_role_arn(self._data_mesh_account_id, DATA_MESH_READONLY_ROLENAME),
+                principal=utils.get_role_arn(account_id=self._data_mesh_account_id,
+                                             role_name=DATA_MESH_READONLY_ROLENAME),
                 database_name=data_mesh_database_name,
                 table_name=table_name,
                 permissions=['DESCRIBE'],
                 grantable_permissions=None
             )
-            self._logger.info(f"Granted Describe on {table_name} to {DATA_MESH_READONLY_ROLENAME}")
+            self._logger.info(f"Granted Describe on Table {table_name} to {DATA_MESH_READONLY_ROLENAME}")
 
         # in the producer account, accept the RAM share after 1 second - seems to be an async delay
-        if permissions_granted > 0:
-            time.sleep(1)
-            self._producer_automator.accept_pending_lf_resource_shares(
-                sender_account=data_mesh_account_id
-            )
+        time.sleep(1)
+        self._producer_automator.accept_pending_lf_resource_shares(
+            sender_account=data_mesh_account_id
+        )
 
-            # create a resource link for the data mesh table in producer account
-            if use_original_table_name is True:
-                link_table_name = table_name
-            else:
-                link_table_name = f"{table_name}_link"
-                if expose_table_references_with_suffix is not None:
-                    link_table_name = f"{table_name}{expose_table_references_with_suffix}"
+        # create a resource link for the data mesh table in producer account
+        if use_original_table_name is True:
+            link_table_name = table_name
+        else:
+            link_table_name = f"{table_name}_link"
+            if expose_table_references_with_suffix is not None:
+                link_table_name = f"{table_name}{expose_table_references_with_suffix}"
 
-            self._producer_automator.create_remote_table(
-                data_mesh_account_id=self._data_mesh_account_id,
-                database_name=data_mesh_database_name,
-                local_table_name=link_table_name,
-                remote_table_name=table_name
-            )
+        self._producer_automator.create_remote_table(
+            data_mesh_account_id=self._data_mesh_account_id,
+            database_name=data_mesh_database_name,
+            local_table_name=link_table_name,
+            remote_table_name=table_name
+        )
 
-            return table_name, link_table_name
+        return table_name, link_table_name
 
     def _make_database_name(self, database_name: str):
         return "%s-%s" % (database_name, self._data_producer_identity.get('Account'))
@@ -227,6 +233,9 @@ def create_data_products(self, source_database_name: str,
                              expose_data_mesh_db_name: str = None,
                              expose_table_references_with_suffix: str = "_link",
                              use_original_table_name: bool = False):
+        if self._log_level == 'DEBUG':
+            self._logger.debug(locals())
+
         if create_public_metadata is None:
             create_public_metadata = True
 
@@ -272,7 +281,7 @@ def create_data_products(self, source_database_name: str,
         # grant the mesh permissions to administer the database
         self._mesh_automator.lf_grant_permissions(
             data_mesh_account_id=self._data_mesh_account_id,
-            principal=self._data_mesh_arn,
+            principal=self._data_producer_role_arn,
             database_name=data_mesh_database_name,
             permissions=['ALL'],
             grantable_permissions=None
@@ -334,19 +343,24 @@ def create_data_products(self, source_database_name: str,
             )
 
             # grant the mesh permissions to describe and select from the table
+            manager_perms = ['DESCRIBE', 'SELECT', 'DROP']
             self._mesh_automator.lf_grant_permissions(
                 data_mesh_account_id=self._data_mesh_account_id,
-                principal=self._data_mesh_arn,
+                principal=utils.get_role_arn(account_id=self._data_mesh_account_id,
+                                             role_name=DATA_MESH_MANAGER_ROLENAME),
                 database_name=data_mesh_database_name,
                 table_name=table.get('Name'),
-                permissions=['DESCRIBE', 'SELECT'],
+                permissions=manager_perms,
                 grantable_permissions=None
             )
             self._logger.info(
-                f"Granted describe access on Table {table.get('Name')} to Data Mesh {self._data_mesh_account_id}")
+                f"Granted {manager_perms} access on Table {table.get('Name')} to {DATA_MESH_MANAGER_ROLENAME}")
 
             shared_objects.get('Tables').append({
-                'SourceTable': created_table[0],
+                'SourceTable': table.get('Name'),
+                'TargetDatabase': data_mesh_database_name,
+                'TargetTable': created_table[0],
+                'LinkDatabase': data_mesh_database_name,
                 'LinkTable': created_table[1]
             })
 

src/data_mesh_util/lib/ApiAutomator.py

@@ -25,6 +25,9 @@ def __init__(self, target_account: str, session: boto3.session.Session, log_leve
         self._logger.setLevel(log_level)
         self._clients = {}
 
+        if log_level == 'DEBUG':
+            utils.log_instance_signature(self, self._logger)
+
     def _get_client(self, client_name):
         client = self._clients.get(client_name)
 
@@ -822,9 +825,16 @@ def create_lf_permissions_entry(self,
                 entry["Id"] = shortuuid.uuid()
 
             log_message = f"{target_account_id} Database {database_name} Permissions:{permissions}"
+
             if grantable_permissions is not None:
-                entry["PermissionsWithGrantOption"] = grantable_permissions
-                log_message = f"{log_message}, {grantable_permissions} WITH GRANT OPTION"
+                # make sure that grantable permissions can't include anything that isn't in permissions
+                _set_grantable_permissions = []
+                for p in permissions:
+                    if p in grantable_permissions:
+                        _set_grantable_permissions.append(p)
+
+                entry["PermissionsWithGrantOption"] = _set_grantable_permissions
+                log_message = f"{log_message}, {_set_grantable_permissions} WITH GRANT OPTION"
 
             self._logger.info(log_message)
             db_entries.append(entry)
@@ -910,7 +920,8 @@ def lf_batch_grant_permissions(self,
             entries.extend(self.create_lf_permissions_entry(
                 data_mesh_account_id=data_mesh_account_id,
                 target_account_id=target_account_id,
-                database_name=database_name, table_name=t,
+                database_name=database_name,
+                table_name=t,
                 permissions=permissions,
                 grantable_permissions=grantable_permissions,
                 target_batch=True)
@@ -927,7 +938,13 @@ def lf_batch_grant_permissions(self,
         if 'Failures' in response:
             perms_added -= len(response.get('Failures'))
 
-        return perms_added
+        if perms_added == 0:
+            self._logger.error(
+                f"Exceptions raised while granting Batch Permissions from {data_mesh_account_id} to {target_account_id}")
+            self._logger.error(response.get('Failures'))
+            raise Exception(f"Failed to grant permissions on Account {data_mesh_account_id}")
+        else:
+            return perms_added
 
     def lf_grant_permissions(self, data_mesh_account_id: str, principal: str, database_name: str,
                              table_name: str = None,

src/data_mesh_util/lib/utils.py

@@ -344,3 +344,8 @@ def generate_resource(service: str, region: str, credentials):
     if 'SessionToken' in use_creds:
         args['aws_session_token'] = use_creds.get('SessionToken')
     return boto3.resource(**args)
+
+def log_instance_signature(obj, logger) -> None:
+    logger.debug(f"Instance Signature for {obj.__class__.__name__}")
+    for attr in vars(obj):
+        logger.debug(f"{attr}: {obj.__dict__.get(attr)}")

src/data_mesh_util/resource/producer_account_policy.pystache

@@ -41,6 +41,7 @@
             "Sid": "ProducerAccess4",
             "Effect": "Allow",
             "Action": [
+                "lakeformation:BatchGrantPermissions",
                 "lakeformation:GrantPermissions",
                 "lakeformation:GetResourceLFTags",
                 "lakeformation:GetLFTag"

src/data_mesh_util/resource/producer_mesh_policy.pystache

@@ -73,7 +73,8 @@
             "Sid": "ProducerPolicy2",
             "Effect": "Allow",
             "Action": [
-                "ram:CreateResourceShare"
+                "ram:CreateResourceShare",
+                "ram:GetResourceShares"
             ],
             "Resource": "*"
         },