diff --git a/.env.template b/.env.template index 9681b38c..f64b403e 100644 --- a/.env.template +++ b/.env.template @@ -89,6 +89,8 @@ SENTRT_TSR= # profile sampling rate SENTRY_PSR= +BIZ_DEV_EMAILS="" + # docker specific FIRST_ORG_NAME="admin-dev" FIRST_USER_EMAIL="admin@gmail.com" diff --git a/ddpui/api/notifications_api.py b/ddpui/api/notifications_api.py index b11170b5..e530282a 100644 --- a/ddpui/api/notifications_api.py +++ b/ddpui/api/notifications_api.py @@ -13,7 +13,7 @@ @notification_router.post("/") -def create_notification(request, payload: CreateNotificationPayloadSchema): +def post_create_notification(request, payload: CreateNotificationPayloadSchema): """Handle the task of creating a notification""" # Filter OrgUser data based on sent_to field diff --git a/ddpui/api/org_preferences_api.py b/ddpui/api/org_preferences_api.py new file mode 100644 index 00000000..d482ff7f --- /dev/null +++ b/ddpui/api/org_preferences_api.py @@ -0,0 +1,220 @@ +import os +from ninja import Router +from ninja.errors import HttpError +from django.utils import timezone +from ddpui import auth +from ddpui.models.org_preferences import OrgPreferences +from ddpui.models.org_supersets import OrgSupersets +from ddpui.models.org_plans import OrgPlans +from ddpui.models.userpreferences import UserPreferences +from ddpui.schemas.org_preferences_schema import ( + CreateOrgPreferencesSchema, + UpdateLLMOptinSchema, + UpdateDiscordNotificationsSchema, + CreateOrgSupersetDetailsSchema, +) +from ddpui.core.notifications_service import create_notification +from ddpui.schemas.notifications_api_schemas import NotificationDataSchema +from django.db import transaction +from ddpui.auth import has_permission +from ddpui.models.org_user import OrgUser +from ddpui.ddpdbt import dbt_service +from ddpui.ddpairbyte import airbyte_service +from ddpui.ddpprefect import ( + prefect_service, +) +from ddpui.utils.awsses import send_text_message + +orgpreference_router = Router() + + +@orgpreference_router.post("/", auth=auth.CustomAuthMiddleware()) +def create_org_preferences(request, payload: CreateOrgPreferencesSchema): + """Creates preferences for an organization""" + orguser: OrgUser = request.orguser + org = orguser.org + payload.org = org + if OrgPreferences.objects.filter(org=org).exists(): + raise HttpError(400, "Organization preferences already exist") + + payload_data = payload.dict(exclude={"org"}) + + org_preferences = OrgPreferences.objects.create( + org=org, **payload_data # Use the rest of the payload + ) + + return {"success": True, "res": org_preferences.to_json()} + + +@orgpreference_router.put("/llm_approval", auth=auth.CustomAuthMiddleware()) +@has_permission(["can_edit_llm_settings"]) +@transaction.atomic +def update_org_preferences(request, payload: UpdateLLMOptinSchema): + """Updates llm preferences for the logged-in user's organization""" + + orguser: OrgUser = request.orguser + org = orguser.org + + org_preferences = OrgPreferences.objects.filter(org=org).first() + user_preferences = UserPreferences.objects.filter(orguser=orguser).first() + + if org_preferences is None: + org_preferences = OrgPreferences.objects.create(org=org) + + if user_preferences is None: + user_preferences = UserPreferences.objects.create(orguser=orguser) + + if payload.llm_optin is True: + org_preferences.llm_optin = True + org_preferences.llm_optin_approved_by = orguser + org_preferences.llm_optin_date = timezone.now() + user_preferences.disclaimer_shown = True + org_preferences.enable_llm_request = False + org_preferences.enable_llm_requested_by = None + else: + org_preferences.llm_optin = False + org_preferences.llm_optin_approved_by = None + org_preferences.llm_optin_date = None + user_preferences.save() + org_preferences.save() + + # sending notification to all users in the org. + if payload.llm_optin is True: + recipients: list[OrgUser] = OrgUser.objects.filter(org=org).all() + + notification_payload = NotificationDataSchema( + author=orguser.user.email, + message="The AI LLM Data Analysis feature is now enabled.", + urgent=False, + scheduled_time=None, + recipients=[recipient.id for recipient in recipients], + ) + + error, res = create_notification(notification_payload) + if res and "errors" in res and len(res["errors"]) > 0: + raise HttpError(400, "Issue with creating the request notification") + + return {"success": True, "res": org_preferences.to_json()} + + +@orgpreference_router.put("/enable-discord-notifications", auth=auth.CustomAuthMiddleware()) +@has_permission(["can_edit_org_notification_settings"]) +def update_discord_notifications(request, payload: UpdateDiscordNotificationsSchema): + """Updates Discord notifications preferences for the logged-in user's organization.""" + + orguser: OrgUser = request.orguser + org = orguser.org + + org_preferences = OrgPreferences.objects.filter(org=org).first() + if org_preferences is None: + org_preferences = OrgPreferences.objects.create(org=org) + + if payload.enable_discord_notifications: + if not org_preferences.discord_webhook and not payload.discord_webhook: + raise HttpError(400, "Discord webhook is required to enable notifications.") + if payload.discord_webhook: + org_preferences.discord_webhook = payload.discord_webhook + org_preferences.enable_discord_notifications = True + else: + org_preferences.discord_webhook = None + org_preferences.enable_discord_notifications = False + + org_preferences.save() + + return {"success": True, "res": org_preferences.to_json()} + + +@orgpreference_router.get("/", auth=auth.CustomAuthMiddleware()) +def get_org_preferences(request): + """Gets preferences for an organization based on the logged-in user's organization""" + orguser: OrgUser = request.orguser + org = orguser.org + + org_preferences = OrgPreferences.objects.filter(org=org).first() + if org_preferences is None: + org_preferences = OrgPreferences.objects.create(org=org) + + return {"success": True, "res": org_preferences.to_json()} + + +@orgpreference_router.get("/toolinfo", auth=auth.CustomAuthMiddleware()) +def get_tools_versions(request): + """get versions of the tools used in the system""" + orguser: OrgUser = request.orguser + org = orguser.org + + org_superset = OrgSupersets.objects.filter(org=org).first() + + versions = [] + + ver = airbyte_service.get_current_airbyte_version() + versions.append({"Airbyte": {"version": ver if ver else "Not available"}}) + + # Prefect Version + ver = prefect_service.get_prefect_version() + versions.append({"Prefect": {"version": ver if ver else "Not available"}}) + + # dbt Version + ver = dbt_service.get_dbt_version(org) + versions.append({"DBT": {"version": ver if ver else "Not available"}}) + + # elementary Version + ver = dbt_service.get_edr_version(org) + versions.append({"Elementary": {"version": ver if ver else "Not available"}}) + + # Superset Version + versions.append( + { + "Superset": { + "version": org_superset.superset_version if org_superset else "Not available" + } + } + ) + + return {"success": True, "res": versions} + + +@orgpreference_router.get("/org-plan", auth=auth.CustomAuthMiddleware()) +def get_org_plans(request): + """Gets preferences for an organization based on the logged-in user's organization""" + orguser: OrgUser = request.orguser + org = orguser.org + + org_plan = OrgPlans.objects.filter(org=org).first() + if org_plan is None: + raise HttpError(400, "Org's Plan not found") + + return {"success": True, "res": org_plan.to_json()} + + +@orgpreference_router.post("/org-plan/upgrade", auth=auth.CustomAuthMiddleware()) +@has_permission(["can_initiate_org_plan_upgrade"]) +def initiate_upgrade_dalgo_plan(request): + """User can click on the upgrade button from the settings panel + which will trigger email to biz dev team""" + orguser: OrgUser = request.orguser + org = orguser.org + + org_plan = OrgPlans.objects.filter(org=org).first() + + if not org_plan: + raise HttpError(400, "Org's Plan not found") + + # trigger emails only once + if org_plan.upgrade_requested: + return {"success": True, "res": "Upgrade request already sent"} + + biz_dev_emails = os.getenv("BIZ_DEV_EMAILS", []).split(",") + + message = "Upgrade plan request from org: {org_name} with plan: {plan_name}".format( + org_name=org.name, plan_name=org_plan.features + ) + subject = "Upgrade plan request from org: {org_name}".format(org_name=org.name) + + for email in biz_dev_emails: + send_text_message(email, subject, message) + + org_plan.upgrade_requested = True + org_plan.save() + + return {"success": True} diff --git a/ddpui/api/user_org_api.py b/ddpui/api/user_org_api.py index 88dba50c..c6e41cd6 100644 --- a/ddpui/api/user_org_api.py +++ b/ddpui/api/user_org_api.py @@ -30,6 +30,7 @@ OrgUserUpdateNewRole, OrgUserUpdatev1, ResetPasswordSchema, + ChangePasswordSchema, UserAttributes, VerifyEmailSchema, ) @@ -38,6 +39,7 @@ from ddpui.utils.deleteorg import delete_warehouse_v1 from ddpui.models.org import OrgWarehouse, Org, OrgType from ddpui.ddpairbyte import airbytehelpers +from ddpui.models.org_preferences import OrgPreferences user_org_router = Router() load_dotenv() @@ -55,7 +57,6 @@ def get_current_user_v2(request, org_slug: str = None): orguser: OrgUser = request.orguser user: User = request.orguser.user org: Org = orguser.org - # warehouse warehouse = OrgWarehouse.objects.filter(org=org).first() curr_orgusers = OrgUser.objects.filter(user=user) @@ -63,6 +64,10 @@ def get_current_user_v2(request, org_slug: str = None): if org_slug: curr_orgusers = curr_orgusers.filter(org__slug=org_slug) + org_preferences = OrgPreferences.objects.filter(org=org).first() + if org_preferences is None: + org_preferences = OrgPreferences.objects.create(org=org) + res = [] for curr_orguser in curr_orgusers.prefetch_related( Prefetch( @@ -85,6 +90,7 @@ def get_current_user_v2(request, org_slug: str = None): ): if curr_orguser.org.orgtncs.exists(): curr_orguser.org.tnc_accepted = curr_orguser.org.orgtncs.exists() + res.append( OrgUserResponse( email=user.email, @@ -99,7 +105,7 @@ def get_current_user_v2(request, org_slug: str = None): for rolep in curr_orguser.new_role.rolepermissions.all() ], is_demo=(curr_orguser.org.type == OrgType.DEMO if curr_orguser.org else False), - llm_optin=curr_orguser.llm_optin, + is_llm_active=org_preferences.llm_optin, ) ) @@ -400,6 +406,18 @@ def post_reset_password(request, payload: ResetPasswordSchema): # pylint: disab return {"success": 1} +@user_org_router.post( + "/users/change_password/", auth=auth.CustomAuthMiddleware() +) # from the settings panel +def change_password(request, payload: ChangePasswordSchema): # pylint: disable=unused-argument + """step 2 of the forgot-password flow""" + orguser = request.orguser + _, error = orguserfunctions.change_password(payload, orguser) + if error: + raise HttpError(400, error) + return {"success": 1} + + @user_org_router.get("/users/verify_email/resend", auth=auth.CustomAuthMiddleware()) @has_permission(["can_resend_email_verification"]) def get_verify_email_resend(request): # pylint: disable=unused-argument diff --git a/ddpui/api/user_preferences_api.py b/ddpui/api/user_preferences_api.py index ba428231..3c7da63b 100644 --- a/ddpui/api/user_preferences_api.py +++ b/ddpui/api/user_preferences_api.py @@ -6,7 +6,12 @@ CreateUserPreferencesSchema, UpdateUserPreferencesSchema, ) +from ddpui.models.org_preferences import OrgPreferences from ddpui.models.org_user import OrgUser +from ddpui.auth import ACCOUNT_MANAGER_ROLE +from ddpui.core.notifications_service import create_notification +from ddpui.auth import has_permission +from ddpui.schemas.notifications_api_schemas import NotificationDataSchema userpreference_router = Router() @@ -19,20 +24,13 @@ def create_user_preferences(request, payload: CreateUserPreferencesSchema): if UserPreferences.objects.filter(orguser=orguser).exists(): raise HttpError(400, "Preferences already exist") - userpreferences = UserPreferences.objects.create( + user_preferences = UserPreferences.objects.create( orguser=orguser, - enable_discord_notifications=payload.enable_discord_notifications, enable_email_notifications=payload.enable_email_notifications, - discord_webhook=payload.discord_webhook, + disclaimer_shown=payload.disclaimer_shown, ) - preferences = { - "discord_webhook": userpreferences.discord_webhook, - "enable_email_notifications": userpreferences.enable_email_notifications, - "enable_discord_notifications": userpreferences.enable_discord_notifications, - } - - return {"success": True, "res": preferences} + return {"success": True, "res": user_preferences.to_json()} @userpreference_router.put("/", auth=auth.CustomAuthMiddleware()) @@ -42,34 +40,65 @@ def update_user_preferences(request, payload: UpdateUserPreferencesSchema): user_preferences, created = UserPreferences.objects.get_or_create(orguser=orguser) - if payload.enable_discord_notifications is not None: - user_preferences.enable_discord_notifications = payload.enable_discord_notifications if payload.enable_email_notifications is not None: user_preferences.enable_email_notifications = payload.enable_email_notifications - if payload.discord_webhook is not None: - user_preferences.discord_webhook = payload.discord_webhook - + if payload.disclaimer_shown is not None: + user_preferences.disclaimer_shown = payload.disclaimer_shown user_preferences.save() - preferences = { - "discord_webhook": user_preferences.discord_webhook, - "enable_email_notifications": user_preferences.enable_email_notifications, - "enable_discord_notifications": user_preferences.enable_discord_notifications, - } - - return {"success": True, "res": preferences} + return {"success": True, "res": user_preferences.to_json()} @userpreference_router.get("/", auth=auth.CustomAuthMiddleware()) def get_user_preferences(request): """gets user preferences for the user""" orguser: OrgUser = request.orguser - user_preferences, created = UserPreferences.objects.get_or_create(orguser=orguser) + org_preferences, created = OrgPreferences.objects.get_or_create(org=orguser.org) - preferences = { - "discord_webhook": user_preferences.discord_webhook, + res = { "enable_email_notifications": user_preferences.enable_email_notifications, - "enable_discord_notifications": user_preferences.enable_discord_notifications, + "disclaimer_shown": user_preferences.disclaimer_shown, + "is_llm_active": org_preferences.llm_optin, + "enable_llm_requested": org_preferences.enable_llm_request, } - return {"success": True, "res": preferences} + return {"success": True, "res": res} + + +@userpreference_router.post("/llm_analysis/request", auth=auth.CustomAuthMiddleware()) +@has_permission(["can_request_llm_analysis_feature"]) +def post_request_llm_analysis_feature_enabled(request): + """Sends a notification to org's account manager for enabling LLM analysis feature""" + orguser: OrgUser = request.orguser + org = orguser.org + + # get the account managers of the org + acc_managers: list[OrgUser] = OrgUser.objects.filter( + org=org, new_role__slug=ACCOUNT_MANAGER_ROLE + ).all() + + if len(acc_managers) == 0: + raise HttpError(400, "No account manager found for the organization") + + # send notification to all account managers + notification_data = NotificationDataSchema( + author=orguser.user.email, + message=f"{orguser.user.email} is requesting to enable LLM analysis feature", + urgent=False, + scheduled_time=None, + recipients=[acc_manager.id for acc_manager in acc_managers], + ) + + error, res = create_notification(notification_data) + if res and "errors" in res and len(res["errors"]) > 0: + raise HttpError(400, "Issue with creating the request notification") + + rows_updated = OrgPreferences.objects.filter(org=org).update( + enable_llm_request=True, enable_llm_requested_by=orguser + ) + if rows_updated == 0: + raise HttpError( + 400, "No rows were updated. OrgPreferences may not exist for this organization." + ) + + return {"success": True, "res": "Notified account manager(s) to enable LLM analysis feature"} diff --git a/ddpui/celeryworkers/tasks.py b/ddpui/celeryworkers/tasks.py index 035b8d85..53a3b70b 100644 --- a/ddpui/celeryworkers/tasks.py +++ b/ddpui/celeryworkers/tasks.py @@ -1,5 +1,6 @@ """these are tasks which we run through celery""" +import pytz import os import shutil from pathlib import Path @@ -18,6 +19,8 @@ from ddpui.utils import timezone, awsses from ddpui.utils.helpers import find_key_in_dictionary from ddpui.utils.custom_logger import CustomLogger +from ddpui.utils.awsses import send_text_message +from ddpui.models.org_plans import OrgPlans from ddpui.models.org import ( Org, OrgDbt, @@ -958,6 +961,32 @@ def summarize_warehouse_results( return +@app.task() +def check_org_plan_expiry_notify_people(): + """detects schema changes for all the orgs and sends an email to admins if there is a change""" + roles_to_notify = [ACCOUNT_MANAGER_ROLE] + days_before_expiry = 7 + + for org in Org.objects.all(): + org_plan = OrgPlans.objects.filter(org=org).first() + if not org_plan: + continue + + # send a notification 7 days before the plan expires + if org_plan.end_date - timedelta(days=days_before_expiry) < datetime.now(pytz.utc): + try: + org_users = OrgUser.objects.filter( + org=org, + new_role__slug__in=roles_to_notify, + ) + message = f"""This email is to let you know that your Dalgo plan is about to expire. Please renew it to continue using the services.""" + subject = "Dalgo plan expiry" + for orguser in org_users: + send_text_message(orguser.user.email, subject, message) + except Exception as err: + logger.error(err) + + @app.task(bind=False) def check_for_long_running_flow_runs(): """checks for long-running flow runs in prefect""" @@ -1024,6 +1053,11 @@ def setup_periodic_tasks(sender, **kwargs): check_for_long_running_flow_runs.s(), name="check for long-running flow-runs", ) + sender.add_periodic_task( + crontab(minute=0, hour="*/12"), + check_org_plan_expiry_notify_people.s(), + name="check org plan expiry and notify the right people", + ) @app.task(bind=True) diff --git a/ddpui/core/notifications_service.py b/ddpui/core/notifications_service.py index 07e96052..fa3ba245 100644 --- a/ddpui/core/notifications_service.py +++ b/ddpui/core/notifications_service.py @@ -11,7 +11,7 @@ from ddpui.utils import timezone from ddpui.utils.discord import send_discord_notification from ddpui.utils.sendgrid import send_email_notification -from ddpui.schemas.notifications_api_schemas import SentToEnum +from ddpui.schemas.notifications_api_schemas import SentToEnum, NotificationDataSchema from ddpui.celeryworkers.tasks import schedule_notification_task @@ -102,18 +102,18 @@ def handle_recipient( # main function for sending notification def create_notification( - notification_data, + notification_data: NotificationDataSchema, ) -> Tuple[Optional[Dict[str, str]], Optional[Dict[str, Any]]]: """ main function for creating notification. Add notification to the notification table. """ - author = notification_data["author"] - message = notification_data["message"] - urgent = notification_data["urgent"] - scheduled_time = notification_data["scheduled_time"] - recipients = notification_data["recipients"] + author = notification_data.author + message = notification_data.message + urgent = notification_data.urgent + scheduled_time = notification_data.scheduled_time + recipients = notification_data.recipients errors = [] notification = Notification.objects.create( diff --git a/ddpui/core/orguserfunctions.py b/ddpui/core/orguserfunctions.py index 05e69a70..b83adf41 100644 --- a/ddpui/core/orguserfunctions.py +++ b/ddpui/core/orguserfunctions.py @@ -27,6 +27,7 @@ OrgUserUpdate, OrgUserUpdatev1, ResetPasswordSchema, + ChangePasswordSchema, UserAttributes, VerifyEmailSchema, ) @@ -153,8 +154,6 @@ def update_orguser_v1(orguser: OrgUser, payload: OrgUserUpdatev1): orguser.user.is_active = payload.active if payload.role_uuid: orguser.new_role = Role.objects.filter(uuid=payload.role_uuid).first() - if payload.llm_optin is not None: - orguser.llm_optin = payload.llm_optin orguser.user.save() orguser.save() @@ -593,6 +592,18 @@ def confirm_reset_password(payload: ResetPasswordSchema): return None, None +def change_password(payload: ChangePasswordSchema, orguser: OrgUser): + """If password and confirm password are same reset the password""" + + if payload.password != payload.confirmPassword: + return None, "Password and confirm password must be same" + + orguser.user.set_password(payload.password.get_secret_value()) + orguser.user.save() + + return None, None + + def resend_verification_email(orguser: OrgUser, email: str): """send a verification email to the user""" redis = RedisClient.get_instance() diff --git a/ddpui/ddpairbyte/airbyte_service.py b/ddpui/ddpairbyte/airbyte_service.py index 069c32ae..bb7a2565 100644 --- a/ddpui/ddpairbyte/airbyte_service.py +++ b/ddpui/ddpairbyte/airbyte_service.py @@ -35,6 +35,10 @@ def abreq(endpoint, req=None, **kwargs): """Request to the airbyte server""" + method = kwargs.get("method", "POST") + if method not in ["GET", "POST"]: + raise HttpError(500, "method not supported") + request = thread.get_current_request() abhost = os.getenv("AIRBYTE_SERVER_HOST") @@ -68,12 +72,21 @@ def abreq(endpoint, req=None, **kwargs): logger.info("Making request to Airbyte server: %s", endpoint) try: - res = requests.post( - f"http://{abhost}:{abport}/api/{abver}/{endpoint}", - headers={"Authorization": f"Basic {token}"}, - json=req, - timeout=kwargs.get("timeout", 30), - ) + res = {} + if method == "POST": + res = requests.post( + f"http://{abhost}:{abport}/api/{abver}/{endpoint}", + headers={"Authorization": f"Basic {token}"}, + json=req, + timeout=kwargs.get("timeout", 30), + ) + elif method == "GET": + res = requests.get( + f"http://{abhost}:{abport}/api/{abver}/{endpoint}", + headers={"Authorization": f"Basic {token}"}, + json=req, + timeout=kwargs.get("timeout", 30), + ) except requests.exceptions.ConnectionError as conn_error: logger.exception(conn_error) raise HttpError(500, str(conn_error)) from conn_error @@ -971,3 +984,14 @@ def update_schema_change( raise HttpError(500, "failed to trigger Prefect flow run") from error return res + + +def get_current_airbyte_version(): + """Fetch airbyte version""" + + res = abreq("instance_configuration", method="GET") + print(res, "AIRBYTE RESPONSE") + if "version" not in res: + logger.error("No version found") + return None + return res["version"] diff --git a/ddpui/ddpdbt/dbt_service.py b/ddpui/ddpdbt/dbt_service.py index 8e17896a..d012d52d 100644 --- a/ddpui/ddpdbt/dbt_service.py +++ b/ddpui/ddpdbt/dbt_service.py @@ -31,6 +31,7 @@ TASK_DBTSEED, TASK_DBTDEPS, ) +from ddpui.core.orgdbt_manager import DbtProjectManager from ddpui.utils.timezone import as_ist from ddpui.utils.custom_logger import CustomLogger from ddpui.utils.redis_client import RedisClient @@ -306,3 +307,36 @@ def refresh_elementary_report_via_prefect(orguser: OrgUser) -> dict: raise HttpError(400, "failed to start a run") from error return res + + +def get_dbt_version(org: Org): + """get dbt version""" + try: + dbt_project_params = DbtProjectManager.gather_dbt_project_params(org, org.dbt) + dbt_version_command = [str(dbt_project_params.dbt_binary), "--version"] + dbt_output = subprocess.check_output(dbt_version_command, text=True) + for line in dbt_output.splitlines(): + if "installed:" in line: + return line.split(":")[1].strip() + return "Not available" + except Exception as err: + logger.error("Error getting dbt version: %s", err) + return "Not available" + + +def get_edr_version(org: Org): + """get elementary report version""" + try: + dbt_project_params = DbtProjectManager.gather_dbt_project_params(org, org.dbt) + elementary_version_command = [ + os.path.join(dbt_project_params.venv_binary, "edr"), + "--version", + ] + elementary_output = subprocess.check_output(elementary_version_command, text=True) + for line in elementary_output.splitlines(): + if line.startswith("Elementary version"): + return line.split()[-1].strip()[:-1] + return "Not available" + except Exception as err: + logger.error("Error getting elementary version: %s", err) + return "Not available" diff --git a/ddpui/ddpprefect/prefect_service.py b/ddpui/ddpprefect/prefect_service.py index f7bed75c..420f691b 100644 --- a/ddpui/ddpprefect/prefect_service.py +++ b/ddpui/ddpprefect/prefect_service.py @@ -159,6 +159,23 @@ def prefect_delete(endpoint: str, **kwargs) -> None: raise HttpError(res.status_code, res.text) from error +# ================================================================================================ +def get_prefect_server_version(): + """fetches the prefect server version""" + try: + prefect_host = os.getenv("PREFECT_SERVER_HOST") + prefect_port = os.getenv("PREFECT_SERVER_PORT") + prefect_url = f"http://{prefect_host}:{prefect_port}/api/admin/version" + prefect_response = requests.get(prefect_url, timeout=5) + if prefect_response.status_code == 200: + version = prefect_response.text.strip().strip('"') + return version + else: + return "Not available" + except Exception: + return "Not available" + + # ================================================================================================ def get_airbyte_server_block_id(blockname) -> str | None: """get the block_id for the server block having this name""" @@ -646,3 +663,9 @@ def get_long_running_flow_runs(nhours: int): """gets long running flow runs from prefect""" flow_runs = prefect_get(f"flow_runs/long-running/{nhours}") return flow_runs["flow_runs"] + + +def get_prefect_version(): + """Fetch secret block id and block name""" + response = prefect_get("prefect/version") + return response diff --git a/ddpui/management/commands/create_notification.py b/ddpui/management/commands/create_notification.py index d32239a2..62b09e5e 100644 --- a/ddpui/management/commands/create_notification.py +++ b/ddpui/management/commands/create_notification.py @@ -5,6 +5,7 @@ from ddpui.schemas.notifications_api_schemas import ( CreateNotificationPayloadSchema, SentToEnum, + NotificationDataSchema, ) @@ -83,13 +84,13 @@ def handle(self, *args, **options): sys.exit(1) # Create notification data - notification_data = { - "author": payload.author, - "message": payload.message, - "urgent": payload.urgent, - "scheduled_time": payload.scheduled_time, - "recipients": recipients, - } + notification_data = NotificationDataSchema( + author=payload.author, + message=payload.message, + urgent=payload.urgent, + scheduled_time=payload.scheduled_time, + recipients=recipients, + ) # Call the create notification service error, result = notifications_service.create_notification(notification_data) diff --git a/ddpui/management/commands/createorgplan.py b/ddpui/management/commands/createorgplan.py new file mode 100644 index 00000000..9069c754 --- /dev/null +++ b/ddpui/management/commands/createorgplan.py @@ -0,0 +1,80 @@ +from datetime import datetime +from django.core.management.base import BaseCommand + +from ddpui.models.org import Org +from ddpui.models.org_plans import OrgPlans + + +class Command(BaseCommand): + """ + This script creates OrgPlans for Orgs + """ + + help = "Create an OrgPlan for an Org" + + def add_arguments(self, parser): + parser.add_argument("--org", type=str, help="Org slug", required=True) + parser.add_argument("--with-superset", action="store_true", help="Include superset") + parser.add_argument( + "--plan", + choices=["Free Trial", "DALGO", "Internal"], + default="DALGO", + ) + parser.add_argument( + "--duration", + choices=["Monthly", "Annual"], + help="Subscription duration", + required=True, + ) + parser.add_argument("--start-date", type=str, help="Start date", required=False) + parser.add_argument("--end-date", type=str, help="End date", required=False) + parser.add_argument("--overwrite", action="store_true", help="Overwrite existing plan") + + def handle(self, *args, **options): + """Create the OrgPlan for the Org""" + org = Org.objects.filter(slug=options["org"]).first() + if org is None: + self.stdout.write(self.style.ERROR(f"Org {options['org']} not found")) + return + + org_plan = OrgPlans.objects.filter(org=org).first() + if org_plan and not options["overwrite"]: + self.stdout.write(self.style.ERROR(f"Org {options['org']} already has a plan")) + return + + if not org_plan: + org_plan = OrgPlans(org=org) + + org_plan.subscription_duration = options["duration"] + + org_plan.start_date = ( + datetime.strptime(options["start_date"], "%Y-%m-%d") if options["start_date"] else None + ) + org_plan.end_date = ( + datetime.strptime(options["end_date"], "%Y-%m-%d") if options["end_date"] else None + ) + + org_plan.features = { + "pipeline": ["Ingest", "Transform", "Orchestrate"], + "aiFeatures": ["AI data analysis"], + "dataQuality": ["Data quality dashboards"], + } + + org_plan.superset_included = options["with_superset"] + if options["with_superset"]: + org_plan.features["superset"] = ["Superset dashboards", "Superset Usage dashboards"] + + org_plan.base_plan = options["plan"] + + if options["plan"] == "Free Trial": + org_plan.can_upgrade_plan = True + + elif options["plan"] == "Internal": + org_plan.can_upgrade_plan = False + + else: + org_plan.can_upgrade_plan = not options["with_superset"] + + org_plan.save() + + print(org_plan.to_json()) diff --git a/ddpui/management/commands/createorgsuperset.py b/ddpui/management/commands/createorgsuperset.py new file mode 100644 index 00000000..32115336 --- /dev/null +++ b/ddpui/management/commands/createorgsuperset.py @@ -0,0 +1,35 @@ +from django.core.management.base import BaseCommand + +from ddpui.models.org import Org +from ddpui.models.org_supersets import OrgSupersets + + +class Command(BaseCommand): + """ + This script creates OrgSupersets for Orgs + """ + + help = "Create an OrgSuperset for an Org" + + def add_arguments(self, parser): + parser.add_argument("--org", type=str, help="Org slug", required=True) + parser.add_argument("--container-name", type=str, help="Container name", required=True) + parser.add_argument("--superset-version", type=str, help="Superset version", required=True) + parser.add_argument("--overwrite", action="store_true", help="Overwrite existing plan") + + def handle(self, *args, **options): + org = Org.objects.get(slug=options["org"]) + + org_superset = OrgSupersets.objects.filter(org=org).first() + if org_superset and not options["overwrite"]: + self.stdout.write(self.style.ERROR(f"Org {options['org']} already has a superset")) + return + + if not org_superset: + org_superset = OrgSupersets(org=org) + + org_superset.container_name = options["container_name"] + org_superset.superset_version = options["superset_version"] + + org_superset.save() + print("OrgSuperset created successfully for " + org.slug) diff --git a/ddpui/management/commands/userpermissions2orgpermissions.py b/ddpui/management/commands/userpermissions2orgpermissions.py new file mode 100644 index 00000000..6bbea0e0 --- /dev/null +++ b/ddpui/management/commands/userpermissions2orgpermissions.py @@ -0,0 +1,65 @@ +from datetime import datetime +from django.core.management.base import BaseCommand + +from ddpui.models.org import Org +from ddpui.models.org_user import OrgUser +from ddpui.models.org_preferences import OrgPreferences +from ddpui.models.userpreferences import UserPreferences + + +class Command(BaseCommand): + """ + This script creates OrgPermissions from UserPermissions + """ + + help = "Create OrgPermissions from UserPermissions" + + def add_arguments(self, parser): + parser.add_argument("--org", type=str, help="Org slug, use 'all' to update all orgs") + + def handle(self, *args, **options): + q = Org.objects + if options["org"] != "all": + q = q.filter(slug=options["org"]) + if q.count() == 0: + print("No orgs found") + return + for org in q.all(): + print("Processing org " + org.slug) + + orgpreferences = OrgPreferences.objects.filter(org=org).first() + if orgpreferences is None: + print("creating org preferences for " + org.slug) + orgpreferences = OrgPreferences.objects.create(org=org) + + for orguser in OrgUser.objects.filter(org=org): + userpreferences = UserPreferences.objects.filter(orguser=orguser).first() + + if userpreferences is not None: + print("Found user preferences for " + orguser.user.email) + + if ( + orgpreferences.llm_optin is False + and userpreferences.disclaimer_shown is True + ): + print("Approving LLM opt-in by " + orguser.user.email) + orgpreferences.llm_optin = True + orgpreferences.llm_optin_approved_by = orguser + orgpreferences.llm_optin_date = datetime.now() + + if ( + orgpreferences.enable_discord_notifications is False + and userpreferences.enable_discord_notifications is True + and userpreferences.discord_webhook is not None + ): + # use the discord webbhook from the first user we find + print( + "Discord notifications enabled by " + + orguser.user.email + + ", settings webook to " + + userpreferences.discord_webhook + ) + orgpreferences.enable_discord_notifications = True + orgpreferences.discord_webhook = userpreferences.discord_webhook + + orgpreferences.save() diff --git a/ddpui/migrations/0093_remove_org_is_demo_org_type.py b/ddpui/migrations/0093_remove_org_is_demo_org_type.py index c37450b5..6b7d4afb 100644 --- a/ddpui/migrations/0093_remove_org_is_demo_org_type.py +++ b/ddpui/migrations/0093_remove_org_is_demo_org_type.py @@ -18,8 +18,8 @@ class Migration(migrations.Migration): model_name="org", name="type", field=models.CharField( - choices=[("demo", "DEMO"), ("poc", "POC"), ("client", "CLIENT")], - default=ddpui.models.org.OrgType["CLIENT"], + choices=[("demo", "DEMO"), ("trial", "TRIAL"), ("subscription", "SUBSCRIPTION")], + default=ddpui.models.org.OrgType["SUBSCRIPTION"], max_length=50, ), ), diff --git a/ddpui/migrations/0104_remove_userpreferences_discord_webhook_and_more.py b/ddpui/migrations/0104_remove_userpreferences_discord_webhook_and_more.py new file mode 100644 index 00000000..913fec20 --- /dev/null +++ b/ddpui/migrations/0104_remove_userpreferences_discord_webhook_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2 on 2024-11-06 15:34 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0103_connectionjob_connectionmeta_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="userpreferences", + name="discord_webhook", + ), + migrations.RemoveField( + model_name="userpreferences", + name="enable_discord_notifications", + ), + ] diff --git a/ddpui/migrations/0105_orgpreferences.py b/ddpui/migrations/0105_orgpreferences.py new file mode 100644 index 00000000..85761f5f --- /dev/null +++ b/ddpui/migrations/0105_orgpreferences.py @@ -0,0 +1,51 @@ +# Generated by Django 4.2 on 2024-11-06 15:41 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0104_remove_userpreferences_discord_webhook_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="OrgPreferences", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("trial_start_date", models.DateTimeField(blank=True, default=None, null=True)), + ("trial_end_date", models.DateTimeField(blank=True, default=None, null=True)), + ("llm_optin", models.BooleanField(default=False)), + ("llm_optin_date", models.DateTimeField(blank=True, null=True)), + ("enable_discord_notifications", models.BooleanField(default=False)), + ("discord_webhook", models.URLField(blank=True, null=True)), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("updated_at", models.DateTimeField(default=django.utils.timezone.now)), + ( + "llm_optin_approved_by", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="approvedby", + to="ddpui.orguser", + ), + ), + ( + "org", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="preferences", + to="ddpui.org", + ), + ), + ], + ), + ] diff --git a/ddpui/migrations/0106_alter_orgpreferences_llm_optin_approved_by.py b/ddpui/migrations/0106_alter_orgpreferences_llm_optin_approved_by.py new file mode 100644 index 00000000..03b0847d --- /dev/null +++ b/ddpui/migrations/0106_alter_orgpreferences_llm_optin_approved_by.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2 on 2024-11-12 08:27 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0105_orgpreferences"), + ] + + operations = [ + migrations.AlterField( + model_name="orgpreferences", + name="llm_optin_approved_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="approvedby", + to="ddpui.orguser", + ), + ), + ] diff --git a/ddpui/migrations/0107_orgsupersets.py b/ddpui/migrations/0107_orgsupersets.py new file mode 100644 index 00000000..845aa262 --- /dev/null +++ b/ddpui/migrations/0107_orgsupersets.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2 on 2024-11-12 17:21 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0106_alter_orgpreferences_llm_optin_approved_by"), + ] + + operations = [ + migrations.CreateModel( + name="OrgSupersets", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("container_name", models.CharField(blank=True, max_length=255, null=True)), + ("superset_version", models.CharField(blank=True, max_length=255, null=True)), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("updated_at", models.DateTimeField(default=django.utils.timezone.now)), + ( + "org", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="orgInfo", + to="ddpui.org", + ), + ), + ], + ), + ] diff --git a/ddpui/migrations/0108_userpreferences_llm_optin.py b/ddpui/migrations/0108_userpreferences_llm_optin.py new file mode 100644 index 00000000..f7950b8c --- /dev/null +++ b/ddpui/migrations/0108_userpreferences_llm_optin.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2 on 2024-11-12 19:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0107_orgsupersets"), + ] + + operations = [ + migrations.AddField( + model_name="userpreferences", + name="llm_optin", + field=models.BooleanField(default=False), + ), + ] diff --git a/ddpui/migrations/0109_alter_orgpreferences_org_alter_orgsupersets_org.py b/ddpui/migrations/0109_alter_orgpreferences_org_alter_orgsupersets_org.py new file mode 100644 index 00000000..7edb6434 --- /dev/null +++ b/ddpui/migrations/0109_alter_orgpreferences_org_alter_orgsupersets_org.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2 on 2024-11-12 20:03 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0108_userpreferences_llm_optin"), + ] + + operations = [ + migrations.AlterField( + model_name="orgpreferences", + name="org", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="preferences", + to="ddpui.org", + ), + ), + migrations.AlterField( + model_name="orgsupersets", + name="org", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, related_name="orgInfo", to="ddpui.org" + ), + ), + ] diff --git a/ddpui/migrations/0110_userpreferences_discord_webhook_and_more.py b/ddpui/migrations/0110_userpreferences_discord_webhook_and_more.py new file mode 100644 index 00000000..27d5d0c1 --- /dev/null +++ b/ddpui/migrations/0110_userpreferences_discord_webhook_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2 on 2024-11-14 08:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0109_alter_orgpreferences_org_alter_orgsupersets_org"), + ] + + operations = [ + migrations.AddField( + model_name="userpreferences", + name="discord_webhook", + field=models.URLField(blank=True, null=True), + ), + migrations.AddField( + model_name="userpreferences", + name="enable_discord_notifications", + field=models.BooleanField(default=False), + ), + ] diff --git a/ddpui/migrations/0111_remove_orgpreferences_trial_end_date_and_more.py b/ddpui/migrations/0111_remove_orgpreferences_trial_end_date_and_more.py new file mode 100644 index 00000000..1e7df27b --- /dev/null +++ b/ddpui/migrations/0111_remove_orgpreferences_trial_end_date_and_more.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2 on 2024-11-15 10:52 + +import ddpui.models.org +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0110_userpreferences_discord_webhook_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="orgpreferences", + name="trial_end_date", + ), + migrations.RemoveField( + model_name="orgpreferences", + name="trial_start_date", + ), + migrations.AlterField( + model_name="org", + name="type", + field=models.CharField( + choices=[("subscription", "SUBSCRIPTION"), ("trial", "TRIAL"), ("demo", "DEMO")], + default=ddpui.models.org.OrgType["SUBSCRIPTION"], + max_length=50, + ), + ), + migrations.CreateModel( + name="OrgPlans", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("base_plan", models.CharField(default=None, max_length=255, null=True)), + ("superset_included", models.BooleanField(default=False, null=True)), + ( + "subscription_duration", + models.CharField(default=None, max_length=255, null=True), + ), + ("features", models.JSONField(default=None, null=True)), + ("start_date", models.DateTimeField(blank=True, default=None, null=True)), + ("end_date", models.DateTimeField(blank=True, default=None, null=True)), + ("can_upgrade_plan", models.BooleanField(default=False)), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ("updated_at", models.DateTimeField(default=django.utils.timezone.now)), + ( + "org", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="org_plans", + to="ddpui.org", + ), + ), + ], + ), + ] diff --git a/ddpui/migrations/0112_rename_llm_optin_userpreferences_disclaimer_shown.py b/ddpui/migrations/0112_rename_llm_optin_userpreferences_disclaimer_shown.py new file mode 100644 index 00000000..431c4cc4 --- /dev/null +++ b/ddpui/migrations/0112_rename_llm_optin_userpreferences_disclaimer_shown.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2 on 2024-11-17 07:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0111_remove_orgpreferences_trial_end_date_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="userpreferences", + old_name="llm_optin", + new_name="disclaimer_shown", + ), + ] diff --git a/ddpui/migrations/0113_orgplans_upgrade_requested_and_more.py b/ddpui/migrations/0113_orgplans_upgrade_requested_and_more.py new file mode 100644 index 00000000..ad8265bf --- /dev/null +++ b/ddpui/migrations/0113_orgplans_upgrade_requested_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 4.2 on 2024-11-20 07:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("ddpui", "0112_rename_llm_optin_userpreferences_disclaimer_shown"), + ] + + operations = [ + migrations.AddField( + model_name="orgplans", + name="upgrade_requested", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="orgpreferences", + name="enable_llm_request", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="orgpreferences", + name="enable_llm_requested_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="llm_request", + to="ddpui.orguser", + ), + ), + ] diff --git a/ddpui/models/__init__.py b/ddpui/models/__init__.py index c0757144..93c260b3 100644 --- a/ddpui/models/__init__.py +++ b/ddpui/models/__init__.py @@ -1,3 +1,5 @@ from ddpui.models.tasks import Task from ddpui.models.llm import AssistantPrompt from ddpui.models.syncstats import SyncStats +from ddpui.models.org_preferences import OrgPreferences +from ddpui.models.org_supersets import OrgSupersets diff --git a/ddpui/models/org.py b/ddpui/models/org.py index 8878f4bb..458b7136 100644 --- a/ddpui/models/org.py +++ b/ddpui/models/org.py @@ -7,9 +7,9 @@ class OrgType(str, Enum): """an enum representing the type of organization""" + SUBSCRIPTION = "subscription" + TRIAL = "trial" DEMO = "demo" - POC = "poc" - CLIENT = "client" @classmethod def choices(cls): @@ -70,7 +70,7 @@ class Org(models.Model): ) viz_url = models.CharField(max_length=100, null=True) viz_login_type = models.CharField(choices=OrgVizLoginType.choices(), max_length=50, null=True) - type = models.CharField(choices=OrgType.choices(), max_length=50, default=OrgType.CLIENT) + type = models.CharField(choices=OrgType.choices(), max_length=50, default=OrgType.SUBSCRIPTION) ses_whitelisted_email = models.TextField(max_length=100, null=True) created_at = models.DateTimeField(auto_created=True, default=timezone.now) updated_at = models.DateTimeField(auto_now=True) diff --git a/ddpui/models/org_plans.py b/ddpui/models/org_plans.py new file mode 100644 index 00000000..9224e320 --- /dev/null +++ b/ddpui/models/org_plans.py @@ -0,0 +1,43 @@ +from django.db import models +from django.utils import timezone +from ddpui.models.org import Org + + +class OrgPlans(models.Model): + """Model to store org preferences for settings panel""" + + org = models.OneToOneField(Org, on_delete=models.CASCADE, related_name="org_plans") + base_plan = models.CharField( + null=True, max_length=255, default=None + ) # plan DALGO or FREE TRAIL + superset_included = models.BooleanField(null=True, default=False) + subscription_duration = models.CharField( + null=True, max_length=255, default=None + ) # monthly, quatery. + features = models.JSONField( + null=True, default=None + ) # put the features as json and map in the frontend. + start_date = models.DateTimeField(null=True, blank=True, default=None) + end_date = models.DateTimeField(null=True, blank=True, default=None) + can_upgrade_plan = models.BooleanField(default=False) + upgrade_requested = models.BooleanField(default=False) + created_at = models.DateTimeField(default=timezone.now) + updated_at = models.DateTimeField(default=timezone.now) + + def to_json(self) -> dict: + """Return a dict representation of the model""" + return { + "org": { + "name": self.org.name, + "slug": self.org.slug, + "type": self.org.type, + }, + "base_plan": self.base_plan, + "superset_included": self.superset_included, + "subscription_duration": self.subscription_duration, + "features": self.features, + "start_date": self.start_date, + "end_date": self.end_date, + "can_upgrade_plan": self.can_upgrade_plan, + "upgrade_requested": self.upgrade_requested, + } diff --git a/ddpui/models/org_preferences.py b/ddpui/models/org_preferences.py new file mode 100644 index 00000000..8d896e85 --- /dev/null +++ b/ddpui/models/org_preferences.py @@ -0,0 +1,40 @@ +from django.db import models +from ddpui.models.org import Org +from ddpui.models.org_user import OrgUser +from django.utils import timezone + + +class OrgPreferences(models.Model): + """Model to store org preferences for settings panel""" + + org = models.OneToOneField(Org, on_delete=models.CASCADE, related_name="preferences") + llm_optin = models.BooleanField(default=False) + llm_optin_approved_by = models.ForeignKey( + OrgUser, on_delete=models.CASCADE, related_name="approvedby", null=True, blank=True + ) + llm_optin_date = models.DateTimeField(null=True, blank=True) + enable_llm_request = models.BooleanField(default=False) + enable_llm_requested_by = models.ForeignKey( + OrgUser, on_delete=models.CASCADE, related_name="llm_request", null=True, blank=True + ) + enable_discord_notifications = models.BooleanField(default=False) + discord_webhook = models.URLField(blank=True, null=True) + created_at = models.DateTimeField(default=timezone.now) + updated_at = models.DateTimeField(default=timezone.now) + + def to_json(self) -> dict: + """Return a dict representation of the model""" + return { + "org": { + "name": self.org.name, + "slug": self.org.slug, + "type": self.org.type, + }, + "llm_optin": bool(self.llm_optin), + "llm_optin_approved_by": ( + self.llm_optin_approved_by.user.email if self.llm_optin_approved_by else None + ), + "llm_optin_date": self.llm_optin_date.isoformat() if self.llm_optin_date else None, + "enable_discord_notifications": bool(self.enable_discord_notifications), + "discord_webhook": self.discord_webhook, + } diff --git a/ddpui/models/org_supersets.py b/ddpui/models/org_supersets.py new file mode 100644 index 00000000..7395b8e1 --- /dev/null +++ b/ddpui/models/org_supersets.py @@ -0,0 +1,13 @@ +from django.db import models +from django.utils import timezone +from ddpui.models.org import Org + + +class OrgSupersets(models.Model): + """Model to store org supereset details for settings panel""" + + org = models.OneToOneField(Org, on_delete=models.CASCADE, related_name="orgInfo") + container_name = models.CharField(max_length=255, blank=True, null=True) + superset_version = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField(default=timezone.now) + updated_at = models.DateTimeField(default=timezone.now) diff --git a/ddpui/models/org_user.py b/ddpui/models/org_user.py index e3cd2bf9..9fc97f58 100644 --- a/ddpui/models/org_user.py +++ b/ddpui/models/org_user.py @@ -70,7 +70,7 @@ class OrgUser(models.Model): role = models.IntegerField(choices=OrgUserRole.choices(), default=OrgUserRole.REPORT_VIEWER) new_role = models.ForeignKey(Role, on_delete=models.SET_NULL, null=True) email_verified = models.BooleanField(default=False) - llm_optin = models.BooleanField(default=False) + llm_optin = models.BooleanField(default=False) # deprecated created_at = models.DateTimeField(auto_created=True, default=timezone.now) updated_at = models.DateTimeField(auto_now=True) @@ -103,7 +103,6 @@ class OrgUserUpdatev1(Schema): role_uuid: uuid.UUID = None email: str = None active: bool = None - llm_optin: bool = False class OrgUserUpdateNewRole(Schema): @@ -131,7 +130,7 @@ class OrgUserResponse(Schema): is_demo: bool = False new_role_slug: str | None permissions: list[dict] - llm_optin: bool = None + is_llm_active: bool = None class Invitation(models.Model): @@ -188,6 +187,13 @@ class ResetPasswordSchema(Schema): password: SecretStr +class ChangePasswordSchema(Schema): + """Reset password from settings pannel""" + + password: SecretStr + confirmPassword: SecretStr + + class VerifyEmailSchema(Schema): """the payload for the verify-email workflow""" diff --git a/ddpui/models/userpreferences.py b/ddpui/models/userpreferences.py index 5e2b84b8..0c36c91a 100644 --- a/ddpui/models/userpreferences.py +++ b/ddpui/models/userpreferences.py @@ -1,14 +1,22 @@ from django.db import models -from ddpui.models.org_user import OrgUser from django.utils import timezone +from ddpui.models.org_user import OrgUser class UserPreferences(models.Model): """Model to store user preferences for notifications""" orguser = models.OneToOneField(OrgUser, on_delete=models.CASCADE, related_name="preferences") - enable_discord_notifications = models.BooleanField(default=False) + enable_discord_notifications = models.BooleanField(default=False) # deprecated + discord_webhook = models.URLField(blank=True, null=True) # deprecated enable_email_notifications = models.BooleanField(default=False) - discord_webhook = models.URLField(blank=True, null=True) + disclaimer_shown = models.BooleanField(default=False) created_at = models.DateTimeField(default=timezone.now) updated_at = models.DateTimeField(default=timezone.now) + + def to_json(self) -> dict: + """Return a dict representation of the model""" + return { + "enable_email_notifications": self.enable_email_notifications, + "disclaimer_shown": self.disclaimer_shown, + } diff --git a/ddpui/routes.py b/ddpui/routes.py index d61d8e5e..4322c986 100644 --- a/ddpui/routes.py +++ b/ddpui/routes.py @@ -15,6 +15,7 @@ from ddpui.api.transform_api import transform_router from ddpui.api.user_org_api import user_org_router from ddpui.api.user_preferences_api import userpreference_router +from ddpui.api.org_preferences_api import orgpreference_router from ddpui.api.warehouse_api import warehouse_router from ddpui.api.webhook_api import webhook_router @@ -71,6 +72,7 @@ def ninja_default_error_handler( transform_router.tags = ["Transform"] user_org_router.tags = ["UserOrg"] userpreference_router.tags = ["UserPreference"] +orgpreference_router.tags = ["OrgPreference"] warehouse_router.tags = ["Warehouse"] webhook_router.tags = ["Webhook"] @@ -89,3 +91,4 @@ def ninja_default_error_handler( src_api.add_router("/api/warehouse/", warehouse_router) src_api.add_router("/api/", user_org_router) src_api.add_router("/webhooks/", webhook_router) +src_api.add_router("/api/orgpreferences/", orgpreference_router) diff --git a/ddpui/schemas/notifications_api_schemas.py b/ddpui/schemas/notifications_api_schemas.py index 367cc252..8793b2b4 100644 --- a/ddpui/schemas/notifications_api_schemas.py +++ b/ddpui/schemas/notifications_api_schemas.py @@ -44,3 +44,13 @@ class UpdateReadStatusSchemav1(Schema): notification_ids: list[int] read_status: bool + + +class NotificationDataSchema(Schema): + """Schema use to call the notification service function for creating a notification""" + + author: str + message: str + urgent: Optional[bool] = False + scheduled_time: Optional[datetime] = None + recipients: List[int] # list of orguser ids diff --git a/ddpui/schemas/org_preferences_schema.py b/ddpui/schemas/org_preferences_schema.py new file mode 100644 index 00000000..c28b52bc --- /dev/null +++ b/ddpui/schemas/org_preferences_schema.py @@ -0,0 +1,48 @@ +from typing import Optional +from datetime import datetime +from ninja import Schema + + +class CreateOrgPreferencesSchema(Schema): + """Schema for creating organization preferences.""" + + org: Optional[int] + trial_start_date: Optional[datetime] + trial_end_date: Optional[datetime] + llm_optin: Optional[bool] = False + llm_optin_approved_by: Optional[int] + llm_optin_date: Optional[datetime] + enable_discord_notifications: Optional[bool] = False + discord_webhook: Optional[str] + + +class UpdateOrgPreferencesSchema(Schema): + """Schema for updating organization preferences.""" + + trial_start_date: Optional[datetime] = None + trial_end_date: Optional[datetime] = None + llm_optin: Optional[bool] = None + llm_optin_approved_by: Optional[int] = None + llm_optin_date: Optional[datetime] = None + enable_discord_notifications: Optional[bool] = None + discord_webhook: Optional[str] = None + + +class UpdateLLMOptinSchema(Schema): + """Schema for updating organization LLM approval preference.""" + + llm_optin: bool + + +class UpdateDiscordNotificationsSchema(Schema): + """Schema for updating organization discord notification settings.""" + + enable_discord_notifications: bool + discord_webhook: Optional[str] + + +class CreateOrgSupersetDetailsSchema(Schema): + """Schema for creating organization superset details.""" + + superset_version: Optional[str] + container_name: Optional[str] diff --git a/ddpui/schemas/userpreferences_schema.py b/ddpui/schemas/userpreferences_schema.py index e9009a40..7651686f 100644 --- a/ddpui/schemas/userpreferences_schema.py +++ b/ddpui/schemas/userpreferences_schema.py @@ -5,14 +5,12 @@ class CreateUserPreferencesSchema(Schema): """Schema for creating user preferences for the user.""" - enable_discord_notifications: bool enable_email_notifications: bool - discord_webhook: Optional[str] = None + disclaimer_shown: Optional[bool] = None class UpdateUserPreferencesSchema(Schema): """Schema for updating user preferences for the user.""" - enable_discord_notifications: Optional[bool] = None enable_email_notifications: Optional[bool] = None - discord_webhook: Optional[str] = None + disclaimer_shown: Optional[bool] = None diff --git a/ddpui/tests/api_tests/test_notifications_api.py b/ddpui/tests/api_tests/test_notifications_api.py index d23e2572..1e1c2ef6 100644 --- a/ddpui/tests/api_tests/test_notifications_api.py +++ b/ddpui/tests/api_tests/test_notifications_api.py @@ -9,7 +9,7 @@ from ddpui import auth from django.contrib.auth.models import User from ddpui.api.notifications_api import ( - create_notification, + post_create_notification, get_notification_history, get_notification_recipients, get_user_notifications, @@ -77,7 +77,7 @@ def test_create_notification_success(mock_create_notification, mock_get_recipien create_notification_payload = CreateNotificationPayloadSchema(**payload) mock_get_recipients.return_value = (None, [1, 2, 3]) mock_create_notification.return_value = (None, {"res": [], "errors": []}) - response = create_notification(MagicMock(), create_notification_payload) + response = post_create_notification(MagicMock(), create_notification_payload) assert isinstance(response["res"], list) assert isinstance(response["errors"], list) mock_get_recipients.assert_called_once_with( @@ -117,7 +117,7 @@ def test_create_notification_no_recipients(mock_create_notification, mock_get_re create_notification_payload = CreateNotificationPayloadSchema(**payload) mock_get_recipients.return_value = ("No users found for the given information", []) with pytest.raises(HttpError) as excinfo: - create_notification(MagicMock(), create_notification_payload) + post_create_notification(MagicMock(), create_notification_payload) assert "No users found for the given information" in str(excinfo.value) mock_get_recipients.assert_called_once_with( payload["sent_to"], @@ -151,7 +151,7 @@ def test_create_notification_no_org_slug(mock_create_notification, mock_get_reci [], ) with pytest.raises(HttpError) as excinfo: - create_notification(MagicMock(), create_notification_payload) + post_create_notification(MagicMock(), create_notification_payload) assert "org_slug is required to sent notification to all org users." in str(excinfo.value) mock_get_recipients.assert_called_once_with( payload["sent_to"], @@ -185,7 +185,7 @@ def test_create_notification_no_user_email(mock_create_notification, mock_get_re [], ) with pytest.raises(HttpError) as excinfo: - create_notification(MagicMock(), create_notification_payload) + post_create_notification(MagicMock(), create_notification_payload) assert "user email is required to sent notification to a user." in str(excinfo.value) mock_get_recipients.assert_called_once_with( payload["sent_to"], @@ -219,7 +219,7 @@ def test_create_notification_user_does_not_exist(mock_create_notification, mock_ [], ) with pytest.raises(HttpError) as excinfo: - create_notification(MagicMock(), create_notification_payload) + post_create_notification(MagicMock(), create_notification_payload) assert "User with the provided email does not exist" in str(excinfo.value) mock_get_recipients.assert_called_once_with( payload["sent_to"], diff --git a/ddpui/tests/api_tests/test_user_preferences_api.py b/ddpui/tests/api_tests/test_user_preferences_api.py index 89f6b6cd..f2332a35 100644 --- a/ddpui/tests/api_tests/test_user_preferences_api.py +++ b/ddpui/tests/api_tests/test_user_preferences_api.py @@ -62,9 +62,7 @@ def user_preferences(orguser): """a pytest fixture which creates the user preferences for the OrgUser""" return UserPreferences.objects.create( orguser=orguser, - enable_discord_notifications=True, enable_email_notifications=True, - discord_webhook="http://example.com/webhook", ) @@ -78,20 +76,14 @@ def test_seed_data(seed_db): def test_create_user_preferences_success(orguser): """tests the success of creating user preferences for the OrgUser""" request = mock_request(orguser) - payload = CreateUserPreferencesSchema( - enable_discord_notifications=True, - enable_email_notifications=True, - discord_webhook="http://example.com/webhook", - ) + payload = CreateUserPreferencesSchema(enable_email_notifications=True, disclaimer_shown=True) response = create_user_preferences(request, payload) # Assertions assert response["success"] is True preferences = response["res"] - assert preferences["discord_webhook"] == "http://example.com/webhook" assert preferences["enable_email_notifications"] is True - assert preferences["enable_discord_notifications"] is True def test_create_user_preferences_already_exists(user_preferences): @@ -101,9 +93,7 @@ def test_create_user_preferences_already_exists(user_preferences): """ request = mock_request(orguser=user_preferences.orguser) payload = CreateUserPreferencesSchema( - enable_discord_notifications=True, enable_email_notifications=True, - discord_webhook="http://example.com/webhook", ) with pytest.raises(HttpError) as excinfo: @@ -116,17 +106,13 @@ def test_update_user_preferences_success(orguser, user_preferences): """tests the success of updating user preferences for the OrgUser""" request = mock_request(orguser) payload = UpdateUserPreferencesSchema( - enable_discord_notifications=False, enable_email_notifications=False, - discord_webhook="http://example.org/webhook", ) response = update_user_preferences(request, payload) assert response["success"] is True updated_preferences = UserPreferences.objects.get(orguser=user_preferences.orguser) - assert updated_preferences.enable_discord_notifications is False assert updated_preferences.enable_email_notifications is False - assert updated_preferences.discord_webhook == "http://example.org/webhook" def test_update_user_preferences_create_success_if_not_exist(orguser): @@ -136,17 +122,13 @@ def test_update_user_preferences_create_success_if_not_exist(orguser): """ request = mock_request(orguser) payload = UpdateUserPreferencesSchema( - enable_discord_notifications=True, enable_email_notifications=True, - discord_webhook="http://example.com/webhook", ) response = update_user_preferences(request, payload) assert response["success"] is True user_preferences = UserPreferences.objects.get(orguser=orguser) - assert user_preferences.enable_discord_notifications is True assert user_preferences.enable_email_notifications is True - assert user_preferences.discord_webhook == "http://example.com/webhook" def test_get_user_preferences_success(orguser, user_preferences): @@ -155,9 +137,10 @@ def test_get_user_preferences_success(orguser, user_preferences): response = get_user_preferences(request) assert response["success"] is True assert response["res"] == { - "discord_webhook": user_preferences.discord_webhook, "enable_email_notifications": user_preferences.enable_email_notifications, - "enable_discord_notifications": user_preferences.enable_discord_notifications, + "disclaimer_shown": user_preferences.disclaimer_shown, + "is_llm_active": False, + "enable_llm_requested": False, } @@ -170,8 +153,9 @@ def test_get_user_preferences_success_if_not_exist(orguser): response = get_user_preferences(request) assert response["success"] is True assert response["res"] == { - "discord_webhook": None, "enable_email_notifications": False, - "enable_discord_notifications": False, + "disclaimer_shown": False, + "is_llm_active": False, + "enable_llm_requested": False, } assert UserPreferences.objects.filter(orguser=orguser).exists() diff --git a/ddpui/tests/core/test_notifications_service.py b/ddpui/tests/core/test_notifications_service.py index 546bf9cd..7d067d43 100644 --- a/ddpui/tests/core/test_notifications_service.py +++ b/ddpui/tests/core/test_notifications_service.py @@ -21,7 +21,7 @@ delete_scheduled_notification, get_unread_notifications_count, ) -from ddpui.schemas.notifications_api_schemas import SentToEnum +from ddpui.schemas.notifications_api_schemas import SentToEnum, NotificationDataSchema from ddpui.tests.api_tests.test_user_org_api import mock_request, seed_db from django.contrib.auth.models import User @@ -209,13 +209,14 @@ def test_handle_recipient_discord_error(mocker: Mock, orguser, unsent_notificati def test_create_notification_success(orguser): - notification_data = { - "author": "test_author", - "message": "test_message", - "urgent": True, - "scheduled_time": None, - "recipients": [orguser.id], - } + notification_data = NotificationDataSchema( + author="test_author", + message="test_message", + urgent=True, + scheduled_time=None, + recipients=[orguser.id], + ) + error, result = create_notification(notification_data) assert error is None assert result is not None diff --git a/seed/002_permissions.json b/seed/002_permissions.json index c5cc5086..e1586324 100644 --- a/seed/002_permissions.json +++ b/seed/002_permissions.json @@ -2,27 +2,42 @@ { "model": "ddpui.Permission", "pk": 1, - "fields": { "name": "Can View Dashboard", "slug": "can_view_dashboard" } + "fields": { + "name": "Can View Dashboard", + "slug": "can_view_dashboard" + } }, { "model": "ddpui.Permission", "pk": 2, - "fields": { "name": "Can View Sources", "slug": "can_view_sources" } + "fields": { + "name": "Can View Sources", + "slug": "can_view_sources" + } }, { "model": "ddpui.Permission", "pk": 3, - "fields": { "name": "Can Create Source", "slug": "can_create_source" } + "fields": { + "name": "Can Create Source", + "slug": "can_create_source" + } }, { "model": "ddpui.Permission", "pk": 4, - "fields": { "name": "Can Edit Source", "slug": "can_edit_source" } + "fields": { + "name": "Can Edit Source", + "slug": "can_edit_source" + } }, { "model": "ddpui.Permission", "pk": 5, - "fields": { "name": "Can View Source", "slug": "can_view_source" } + "fields": { + "name": "Can View Source", + "slug": "can_view_source" + } }, { "model": "ddpui.Permission", @@ -35,7 +50,10 @@ { "model": "ddpui.Permission", "pk": 7, - "fields": { "name": "Can View Warehouse", "slug": "can_view_warehouse" } + "fields": { + "name": "Can View Warehouse", + "slug": "can_view_warehouse" + } }, { "model": "ddpui.Permission", @@ -48,7 +66,10 @@ { "model": "ddpui.Permission", "pk": 9, - "fields": { "name": "Can Edit Warehouse", "slug": "can_edit_warehouse" } + "fields": { + "name": "Can Edit Warehouse", + "slug": "can_edit_warehouse" + } }, { "model": "ddpui.Permission", @@ -61,7 +82,10 @@ { "model": "ddpui.Permission", "pk": 11, - "fields": { "name": "Can Create Org", "slug": "can_create_org" } + "fields": { + "name": "Can Create Org", + "slug": "can_create_org" + } }, { "model": "ddpui.Permission", @@ -106,17 +130,26 @@ { "model": "ddpui.Permission", "pk": 17, - "fields": { "name": "Can Delete Source", "slug": "can_delete_source" } + "fields": { + "name": "Can Delete Source", + "slug": "can_delete_source" + } }, { "model": "ddpui.Permission", "pk": 18, - "fields": { "name": "View Tasks", "slug": "can_view_master_tasks" } + "fields": { + "name": "View Tasks", + "slug": "can_view_master_tasks" + } }, { "model": "ddpui.Permission", "pk": 19, - "fields": { "name": "View Task Config", "slug": "can_view_master_task" } + "fields": { + "name": "View Task Config", + "slug": "can_view_master_task" + } }, { "model": "ddpui.Permission", @@ -153,7 +186,10 @@ { "model": "ddpui.Permission", "pk": 24, - "fields": { "name": "Can Run Orgtask", "slug": "can_run_orgtask" } + "fields": { + "name": "Can Run Orgtask", + "slug": "can_run_orgtask" + } }, { "model": "ddpui.Permission", @@ -166,17 +202,26 @@ { "model": "ddpui.Permission", "pk": 26, - "fields": { "name": "Can Create Orgtask", "slug": "can_create_orgtask" } + "fields": { + "name": "Can Create Orgtask", + "slug": "can_create_orgtask" + } }, { "model": "ddpui.Permission", "pk": 27, - "fields": { "name": "Can View Orgtasks", "slug": "can_view_orgtasks" } + "fields": { + "name": "Can View Orgtasks", + "slug": "can_view_orgtasks" + } }, { "model": "ddpui.Permission", "pk": 28, - "fields": { "name": "Can Delete Orgtask", "slug": "can_delete_orgtask" } + "fields": { + "name": "Can Delete Orgtask", + "slug": "can_delete_orgtask" + } }, { "model": "ddpui.Permission", @@ -189,12 +234,18 @@ { "model": "ddpui.Permission", "pk": 30, - "fields": { "name": "Can View Pipelines", "slug": "can_view_pipelines" } + "fields": { + "name": "Can View Pipelines", + "slug": "can_view_pipelines" + } }, { "model": "ddpui.Permission", "pk": 31, - "fields": { "name": "Can View Pipeline", "slug": "can_view_pipeline" } + "fields": { + "name": "Can View Pipeline", + "slug": "can_view_pipeline" + } }, { "model": "ddpui.Permission", @@ -207,12 +258,18 @@ { "model": "ddpui.Permission", "pk": 33, - "fields": { "name": "Can Edit Pipeline", "slug": "can_edit_pipeline" } + "fields": { + "name": "Can Edit Pipeline", + "slug": "can_edit_pipeline" + } }, { "model": "ddpui.Permission", "pk": 34, - "fields": { "name": "Can Run Pipeline", "slug": "can_run_pipeline" } + "fields": { + "name": "Can Run Pipeline", + "slug": "can_run_pipeline" + } }, { "model": "ddpui.Permission", @@ -233,7 +290,10 @@ { "model": "ddpui.Permission", "pk": 37, - "fields": { "name": "Can Sync Sources", "slug": "can_sync_sources" } + "fields": { + "name": "Can Sync Sources", + "slug": "can_sync_sources" + } }, { "model": "ddpui.Permission", @@ -262,7 +322,10 @@ { "model": "ddpui.Permission", "pk": 41, - "fields": { "name": "Can Edit Dbt Model", "slug": "can_edit_dbt_model" } + "fields": { + "name": "Can Edit Dbt Model", + "slug": "can_edit_dbt_model" + } }, { "model": "ddpui.Permission", @@ -299,7 +362,10 @@ { "model": "ddpui.Permission", "pk": 46, - "fields": { "name": "Can View Org Users", "slug": "can_view_orgusers" } + "fields": { + "name": "Can View Org Users", + "slug": "can_view_orgusers" + } }, { "model": "ddpui.Permission", @@ -320,7 +386,10 @@ { "model": "ddpui.Permission", "pk": 49, - "fields": { "name": "Can Edit Org User", "slug": "can_edit_orguser" } + "fields": { + "name": "Can Edit Org User", + "slug": "can_edit_orguser" + } }, { "model": "ddpui.Permission", @@ -341,7 +410,10 @@ { "model": "ddpui.Permission", "pk": 52, - "fields": { "name": "Public", "slug": "public" } + "fields": { + "name": "Public", + "slug": "public" + } }, { "model": "ddpui.Permission", @@ -398,5 +470,37 @@ "name": "Can view feature flags of an organization", "slug": "can_view_flags" } + }, + { + "model": "ddpui.Permission", + "pk": 60, + "fields": { + "name": "Can Edit LLM settings", + "slug": "can_edit_llm_settings" + } + }, + { + "model": "ddpui.Permission", + "pk": 61, + "fields": { + "name": "Can edit org level notifications settings like discord", + "slug": "can_edit_org_notification_settings" + } + }, + { + "model": "ddpui.Permission", + "pk": 62, + "fields": { + "name": "Can initiate upgrade of Dalgo subscription for the org", + "slug": "can_initiate_org_plan_upgrade" + } + }, + { + "model": "ddpui.Permission", + "pk": 63, + "fields": { + "name": "Can request (to the higher role) llm analysis feature to be enabled", + "slug": "can_request_llm_analysis_feature" + } } -] +] \ No newline at end of file diff --git a/seed/003_role_permissions.json b/seed/003_role_permissions.json index 961fd4a4..c477970b 100644 --- a/seed/003_role_permissions.json +++ b/seed/003_role_permissions.json @@ -1774,5 +1774,77 @@ "role": 2, "permission": 58 } + }, + { + "model": "ddpui.RolePermission", + "pk": 219, + "fields": { + "role": 2, + "permission": 60 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 220, + "fields": { + "role": 1, + "permission": 60 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 221, + "fields": { + "role": 2, + "permission": 61 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 222, + "fields": { + "role": 1, + "permission": 61 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 223, + "fields": { + "role": 1, + "permission": 62 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 224, + "fields": { + "role": 2, + "permission": 62 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 225, + "fields": { + "role": 3, + "permission": 63 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 226, + "fields": { + "role": 4, + "permission": 63 + } + }, + { + "model": "ddpui.RolePermission", + "pk": 227, + "fields": { + "role": 5, + "permission": 63 + } } ] \ No newline at end of file