From eb2c59ad46a873618e908bdaecc0cfbb57b89f4a Mon Sep 17 00:00:00 2001 From: Ishan Date: Tue, 9 Jan 2024 13:22:00 +0530 Subject: [PATCH] maintaining single set of usage dashboard creds for all orgs; picking up from the env --- .env.template | 1 + ddpui/api/superset_api.py | 15 ++++++++++++++- .../commands/createsupersetusageuser.py | 16 ++-------------- .../0039_remove_orgwarehouse_superset_creds.py | 16 ++++++++++++++++ ddpui/models/org.py | 3 --- ddpui/tests/helper/test_secretsmanager.py | 14 +++++--------- ddpui/utils/secretsmanager.py | 18 ++++++++---------- 7 files changed, 46 insertions(+), 37 deletions(-) create mode 100644 ddpui/migrations/0039_remove_orgwarehouse_superset_creds.py diff --git a/.env.template b/.env.template index a944a258..e3761fec 100644 --- a/.env.template +++ b/.env.template @@ -43,6 +43,7 @@ SENDGRID_YOUVE_BEEN_ADDED_TEMPLATE= PREFECT_NOTIFICATIONS_WEBHOOK_KEY= SUPERSET_USAGE_DASHBOARD_API_URL= +SUPERSET_USAGE_CREDS_SECRET_ID= diff --git a/ddpui/api/superset_api.py b/ddpui/api/superset_api.py index 625ac374..b273fd9a 100644 --- a/ddpui/api/superset_api.py +++ b/ddpui/api/superset_api.py @@ -63,9 +63,22 @@ def post_fetch_embed_token(request, dashboard_uuid): # pylint: disable=unused-a if warehouse is None: raise HttpError(400, "create a warehouse first") + if orguser.org.viz_url is None: + raise HttpError( + 400, + "your superset subscription is not active, please contact the Dalgo team", + ) + + superset_creds = os.getenv("SUPERSET_USAGE_CREDS_SECRET_ID") + if superset_creds is None: + raise HttpError( + 400, + "superset usage credentials are missing", + ) + # {username: "", password: "", first_name: "", last_name: ""} # skipcq: PY-W0069 credentials = secretsmanager.retrieve_superset_usage_dashboard_credentials( - warehouse + superset_creds ) if credentials is None: raise HttpError(400, "superset usage credentials are missing") diff --git a/ddpui/management/commands/createsupersetusageuser.py b/ddpui/management/commands/createsupersetusageuser.py index 3d98b037..7fd0dcd4 100644 --- a/ddpui/management/commands/createsupersetusageuser.py +++ b/ddpui/management/commands/createsupersetusageuser.py @@ -26,18 +26,8 @@ def add_arguments(self, parser): # skipcq: PYL-R0201 parser.add_argument("--password", required=True) def handle(self, *args, **options): - """adds credentials to secrets manager""" - org = Org.objects.filter(slug=options["org"]).first() - if not org: - logger.error("org not found") - return - warehouse = OrgWarehouse.objects.filter(org=org).first() - if not warehouse: - logger.error("warehouse not found") - return - + """adds superset credentials to secrets manager""" secret_id = save_superset_usage_dashboard_credentials( - warehouse, { "username": options["username"], "first_name": options["first_name"], @@ -45,6 +35,4 @@ def handle(self, *args, **options): "password": options["password"], }, ) - warehouse.superset_creds = secret_id - warehouse.save() - logger.info("credentials saved") + logger.info(f"credentials saved to secretId = {secret_id}") diff --git a/ddpui/migrations/0039_remove_orgwarehouse_superset_creds.py b/ddpui/migrations/0039_remove_orgwarehouse_superset_creds.py new file mode 100644 index 00000000..edc942b9 --- /dev/null +++ b/ddpui/migrations/0039_remove_orgwarehouse_superset_creds.py @@ -0,0 +1,16 @@ +# Generated by Django 4.1.7 on 2024-01-09 07:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0038_org_is_demo"), + ] + + operations = [ + migrations.RemoveField( + model_name="orgwarehouse", + name="superset_creds", + ), + ] diff --git a/ddpui/models/org.py b/ddpui/models/org.py index 7b50635b..678b9c0c 100644 --- a/ddpui/models/org.py +++ b/ddpui/models/org.py @@ -130,9 +130,6 @@ class OrgWarehouse(models.Model): airbyte_norm_op_id = models.TextField( # skipcq: PTC-W0901, PTC-W0906 max_length=36, null=True ) - superset_creds = models.CharField( # skipcq: PTC-W0901, PTC-W0906 - max_length=200, null=True - ) def __str__(self) -> str: return ( diff --git a/ddpui/tests/helper/test_secretsmanager.py b/ddpui/tests/helper/test_secretsmanager.py index 5fb0edbd..3ec5ab7e 100644 --- a/ddpui/tests/helper/test_secretsmanager.py +++ b/ddpui/tests/helper/test_secretsmanager.py @@ -32,9 +32,9 @@ def test_generate_warehouse_credentials_name(): @patch("ddpui.utils.secretsmanager.uuid4", Mock(return_value="unique-val")) def test_generate_superset_credentials_name(): - org = Org(name="temporg", slug="temporg") - response = generate_superset_credentials_name(org) - assert response == "supersetCreds-temporg-unique-val" + response = generate_superset_credentials_name() + assert response == "supersetCreds-unique-val" + assert response.startswith("supersetCreds") @patch( @@ -133,10 +133,7 @@ def test_delete_warehouse_credentials( def test_save_superset_usage_dashboard_credentials(mock_getclient: Mock): create_secret_mock = Mock(return_value={"Name": "supersetcreds"}) mock_getclient.return_value = Mock(create_secret=create_secret_mock) - warehouse = Mock(org=Mock()) - response = save_superset_usage_dashboard_credentials( - warehouse, {"credkey": "credval"} - ) + response = save_superset_usage_dashboard_credentials({"credkey": "credval"}) assert response == "supersetcreds" create_secret_mock.assert_called_once_with( Name="supersetcreds", SecretString='{"credkey": "credval"}' @@ -149,6 +146,5 @@ def test_retrieve_superset_usage_dashboard_credentials( ): get_secret_value = Mock(return_value={"SecretString": '{"credkey": "credval"}'}) mock_getclient.return_value = Mock(get_secret_value=get_secret_value) - warehouse = Mock(superset_creds="credentialskey") - retrieve_superset_usage_dashboard_credentials(warehouse) + retrieve_superset_usage_dashboard_credentials("credentialskey") get_secret_value.assert_called_once_with(SecretId="credentialskey") diff --git a/ddpui/utils/secretsmanager.py b/ddpui/utils/secretsmanager.py index e59fa68b..16af9787 100644 --- a/ddpui/utils/secretsmanager.py +++ b/ddpui/utils/secretsmanager.py @@ -99,9 +99,9 @@ def generate_warehouse_credentials_name(org: Org): return f"warehouseCreds-{org.slug}-{uuid4()}" -def generate_superset_credentials_name(org: Org): +def generate_superset_credentials_name(): """store the connection credentials to their data warehouse""" - return f"supersetCreds-{org.slug}-{uuid4()}" + return f"supersetCreds-{uuid4()}" def save_github_token(org: Org, access_token: str): @@ -196,12 +196,10 @@ def delete_warehouse_credentials(warehouse: OrgWarehouse) -> None: pass -def save_superset_usage_dashboard_credentials( - warehouse: OrgWarehouse, credentials: dict -): - """saves superset usage dashboard user credentials for an org under a predefined secret name""" +def save_superset_usage_dashboard_credentials(credentials: dict): + """saves superset usage dashboard user credentials under a predefined secret name""" aws_sm = get_client() - secret_name = generate_superset_credentials_name(warehouse.org) + secret_name = generate_superset_credentials_name() response = aws_sm.create_secret( Name=secret_name, SecretString=json.dumps(credentials), @@ -214,11 +212,11 @@ def save_superset_usage_dashboard_credentials( def retrieve_superset_usage_dashboard_credentials( - warehouse: OrgWarehouse, + secret_id: str, ) -> dict | None: - """decodes and returns the saved superset usage dashboard credentials for an org""" + """decodes and returns the saved superset usage dashboard credentials""" aws_sm = get_client() - response = aws_sm.get_secret_value(SecretId=warehouse.superset_creds) + response = aws_sm.get_secret_value(SecretId=secret_id) return ( json.loads(response["SecretString"]) if response and "SecretString" in response