From 1798f4c64df58fa093781a5088b516a21ff90903 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 15:58:13 +0530 Subject: [PATCH 001/144] [CU - #1ykyu7k] feat: add new model for Query Change Log --- explorer/models.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/explorer/models.py b/explorer/models.py index 93f38330..2a4c471c 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -112,6 +112,22 @@ class Meta: ordering = ['-run_at'] +class QueryLog(models.Model): + + old_sql = models.TextField(null=True, blank=True) + new_sql = models.TextField(null=True, blank=True) + query = models.ForeignKey(Query, null=True, blank=True, on_delete=models.SET_NULL) + run_by_user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True) + run_at = models.DateTimeField(auto_now_add=True) + + @property + def is_playground(self): + return self.query_id is None + + class Meta: + ordering = ['-run_at'] + + class QueryResult(object): def __init__(self, sql): From 177cb928bbb331685e063bef2477bd486b47facc Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 15:58:56 +0530 Subject: [PATCH 002/144] [CU - #1ykyu7k] feat: added new view for query change logs --- explorer/utils.py | 9 ++++++++- explorer/views.py | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/explorer/utils.py b/explorer/utils.py index 5cf2541a..4d998410 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -236,4 +236,11 @@ def get_s3_connection(): import tinys3 return tinys3.Connection(app_settings.S3_ACCESS_KEY, app_settings.S3_SECRET_KEY, - default_bucket=app_settings.S3_BUCKET) \ No newline at end of file + default_bucket=app_settings.S3_BUCKET) + +def compare_sql(old_sql, new_sql): + """ + Compares whether two sql queries are the + same after formatting them + """ + return fmt_sql(old_sql) == fmt_sql(new_sql) diff --git a/explorer/views.py b/explorer/views.py index 419525de..0d738bbb 100644 --- a/explorer/views.py +++ b/explorer/views.py @@ -16,7 +16,7 @@ from django.db.models import Count from django.forms import ValidationError -from explorer.models import Query, QueryLog, MSG_FAILED_BLACKLIST +from explorer.models import Query, QueryLog, QueryChangeLog, MSG_FAILED_BLACKLIST from explorer import app_settings from explorer.forms import QueryForm from explorer.utils import url_get_rows,\ @@ -30,7 +30,7 @@ user_can_see_query,\ fmt_sql,\ allowed_query_pks,\ - url_get_show + url_get_show, compare_sql try: from collections import Counter @@ -223,6 +223,22 @@ def get_queryset(self): paginate_by = 20 +class ListQueryChangeLogView(ExplorerContextMixin, ListView): + + @method_decorator(view_permission) + def dispatch(self, *args, **kwargs): + return super(ListQueryChangeLogView, self).dispatch(*args, **kwargs) + + def get_queryset(self): + kwargs = {'old_sql__isnumm': False, 'new_sql__isnull': False} + return QueryChangeLog.objects.filter(**kwargs).all() + + context_object_name = "recent_change_logs" + model = QueryChangeLog + paginate_by = 20 + template_name = 'explorer/querychangelog_list.html' + + class CreateQueryView(ExplorerContextMixin, CreateView): @method_decorator(change_permission) @@ -302,7 +318,20 @@ def post(self, request, query_id): ) query, form = QueryView.get_instance_and_form(request, query_id) - success = form.is_valid() and form.save() + old_sql = query.sql; + # success = form.is_valid() and form.save() + form_isvalid = form.is_valid() + if form_isvalid: + new_sql = request.POST.get('sql') + if not compare_sql(old_sql, new_sql): + change_log = QueryChangeLog( + old_sql=old_sql, + new_sql=new_sql, + query=query, + run_by_user=request.user, + ) + change_log.save() + success = form_isvalid and form.save() vm = query_viewmodel(request, query, form=form, message="Query saved." if success else None) return self.render_template('explorer/query.html', vm) From d4ffe8810ebf0b02c8d8eba9941fec611e2bc7fb Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 16:01:38 +0530 Subject: [PATCH 003/144] [CU - #1ykyu7k] feat: added links for query change log template view --- explorer/templates/explorer/play.html | 1 + explorer/templates/explorer/query.html | 1 + explorer/templates/explorer/query_list.html | 1 + .../explorer/querychangelog_list.html | 54 +++++++++++++++++++ .../templates/explorer/querylog_list.html | 1 + explorer/urls.py | 2 + 6 files changed, 60 insertions(+) create mode 100644 explorer/templates/explorer/querychangelog_list.html diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 4e0e3e49..0a337791 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -5,6 +5,7 @@
  • New Query
  • Playground
  • Logs
    • +
  • Change Logs
    • {% endif %} {% endblock %} diff --git a/explorer/templates/explorer/query.html b/explorer/templates/explorer/query.html index 7ffcd3f2..d9a632aa 100644 --- a/explorer/templates/explorer/query.html +++ b/explorer/templates/explorer/query.html @@ -7,6 +7,7 @@ {% endif %} {% if query %}
  • Query Detail
    • {% endif %}
  • Logs
    • +
  • Change Logs
    • {% endblock %} {% block sql_explorer_content %} diff --git a/explorer/templates/explorer/query_list.html b/explorer/templates/explorer/query_list.html index a0e7366d..29775a71 100644 --- a/explorer/templates/explorer/query_list.html +++ b/explorer/templates/explorer/query_list.html @@ -6,6 +6,7 @@
  • New Query
  • Playground
  • Logs
    • +
  • Change Logs
    • {% endif %} {% endblock %} diff --git a/explorer/templates/explorer/querychangelog_list.html b/explorer/templates/explorer/querychangelog_list.html new file mode 100644 index 00000000..27e73187 --- /dev/null +++ b/explorer/templates/explorer/querychangelog_list.html @@ -0,0 +1,54 @@ +{% extends "explorer/base.html" %} + +{% block sql_explorer_navlinks %} + {% if can_change %} +
  • New Query
    • +
  • Playground
    • +
  • Logs
    • +
  • Change Logs
    • + {% endif %} +{% endblock %} + +{% block sql_explorer_content %} +

    Recent Changed Query Logs - Page {{ page_obj.number }}

    + + + + + + + + + + + + + {% for object in recent_logs %} + + + + + + + + + {% endfor %} + +
    Run AtRun ByOld SQLNew SQLQuery IDPlayground
    {{ object.run_at|date:"SHORT_DATETIME_FORMAT" }}{{ object.run_by_user.email }}{{ object.old_sql }}{{ object.new_sql }} {% if object.query_id %}Query {{ object.query_id }}{% elif object.is_playground %}Playground{% else %}--{% endif %}Open
    + {% if is_paginated %} + + {% endif %} +{% endblock %} + diff --git a/explorer/templates/explorer/querylog_list.html b/explorer/templates/explorer/querylog_list.html index 76e66c56..d40a4fa4 100644 --- a/explorer/templates/explorer/querylog_list.html +++ b/explorer/templates/explorer/querylog_list.html @@ -5,6 +5,7 @@
  • New Query
  • Playground
  • Logs
    • +
  • Change Logs
    • {% endif %} {% endblock %} diff --git a/explorer/urls.py b/explorer/urls.py index d270d96c..e88dadbe 100644 --- a/explorer/urls.py +++ b/explorer/urls.py @@ -6,6 +6,7 @@ DeleteQueryView, ListQueryView, ListQueryLogView, + ListQueryChangeLogView, download_query, view_csv_query, email_csv_query, @@ -24,6 +25,7 @@ url(r'play/$', PlayQueryView.as_view(), name='explorer_playground'), url(r'csv$', download_csv_from_sql, name='generate_csv'), url(r'schema/$', schema, name='explorer_schema'), + url(r'changelogs/$', ListQueryChangeLogView.as_view(), name='explorer_change_logs'), url(r'logs/$', ListQueryLogView.as_view(), name='explorer_logs'), url(r'format/$', format_sql, name='format_sql'), url(r'^$', ListQueryView.as_view(), name='explorer_index'), From af283c5078e2b7419322225b3bfb6aa340b2c6d5 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 16:02:05 +0530 Subject: [PATCH 004/144] [CU - #1ykyu7k] feat: added new tests for query change log view --- explorer/tests/test_views.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/explorer/tests/test_views.py b/explorer/tests/test_views.py index 6fc6aafd..e93b788e 100644 --- a/explorer/tests/test_views.py +++ b/explorer/tests/test_views.py @@ -426,4 +426,33 @@ def test_is_playground(self): self.assertTrue(QueryLog(sql='foo').is_playground) q = SimpleQueryFactory() - self.assertFalse(QueryLog(sql='foo', query_id=q.id).is_playground) \ No newline at end of file + self.assertFalse(QueryLog(sql='foo', query_id=q.id).is_playground) + + +class TestQueryChangeLog(TestCase): + + def setUp(self): + self.user = User.objects.create_superuser('admin', 'admin@admin.com', 'pwd') + self.client.login(username='admin', password='pwd') + + def test_admin_required(self): + self.client.logout() + resp = self.client.get(reverse("explorer_change_logs")) + self.assertTemplateUsed(resp, 'admin/login.html') + + def test_query_change_saves_to_change_log(self): + query = SimpleQueryFactory() + data = model_to_dict(query) + data['sql'] = 'select 12345;' + self.client.post(reverse("query_detail", kwargs={'query_id': query.id}), data) + data['sql'] = 'select 67890;' + self.client.post(reverse("query_detail", kwargs={'query_id': query.id}), data) + resp = self.client.get(reverse("explorer_change_logs")) + self.assertContains(resp, 'select 12345;') + self.assertContains(resp, 'select 67890;') + + def test_is_playground(self): + self.assertTrue(QueryChangeLog(old_sql='foo', new_sql='bar').is_playground) + + q = SimpleQueryFactory() + self.assertFalse(QueryChangeLog(old_sql='foo', new_sql='bar', query_id=q.id).is_playground) From 8ab406d79f229838403a684ecdd52bcd2e32024e Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 16:32:16 +0530 Subject: [PATCH 005/144] [CU - #1ykyu7k] feat: added migrations for query change log model --- explorer/migrations/0005_querychangelog.py | 32 ++++++++++++++++++++++ explorer/models.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 explorer/migrations/0005_querychangelog.py diff --git a/explorer/migrations/0005_querychangelog.py b/explorer/migrations/0005_querychangelog.py new file mode 100644 index 00000000..de025a71 --- /dev/null +++ b/explorer/migrations/0005_querychangelog.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2022-03-25 05:46 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('explorer', '0004_querylog_duration'), + ] + + operations = [ + migrations.CreateModel( + name='QueryChangeLog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('old_sql', models.TextField(blank=True, null=True)), + ('new_sql', models.TextField(blank=True, null=True)), + ('run_at', models.DateTimeField(auto_now_add=True)), + ('query', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='explorer.Query')), + ('run_by_user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-run_at'], + }, + ), + ] diff --git a/explorer/models.py b/explorer/models.py index 2a4c471c..d2b8b962 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -112,7 +112,7 @@ class Meta: ordering = ['-run_at'] -class QueryLog(models.Model): +class QueryChangeLog(models.Model): old_sql = models.TextField(null=True, blank=True) new_sql = models.TextField(null=True, blank=True) From 8fc95b3ae101c548e795b2b332790d85c0aba8e7 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 16:33:41 +0530 Subject: [PATCH 006/144] [CU - #1ykyu7k] refactor: code clean-up --- explorer/templates/explorer/querychangelog_list.html | 2 +- explorer/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/explorer/templates/explorer/querychangelog_list.html b/explorer/templates/explorer/querychangelog_list.html index 27e73187..5734cc5f 100644 --- a/explorer/templates/explorer/querychangelog_list.html +++ b/explorer/templates/explorer/querychangelog_list.html @@ -23,7 +23,7 @@

    Recent Changed Query Logs - Page {{ page_obj.number }}

    - {% for object in recent_logs %} + {% for object in recent_change_logs %} {{ object.run_at|date:"SHORT_DATETIME_FORMAT" }} {{ object.run_by_user.email }} diff --git a/explorer/views.py b/explorer/views.py index 0d738bbb..c74914cc 100644 --- a/explorer/views.py +++ b/explorer/views.py @@ -230,7 +230,7 @@ def dispatch(self, *args, **kwargs): return super(ListQueryChangeLogView, self).dispatch(*args, **kwargs) def get_queryset(self): - kwargs = {'old_sql__isnumm': False, 'new_sql__isnull': False} + kwargs = {'old_sql__isnull': False, 'new_sql__isnull': False} return QueryChangeLog.objects.filter(**kwargs).all() context_object_name = "recent_change_logs" From 2042cba277cd0f2d1b81d7aa718ef44cf5dabea8 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 16:34:01 +0530 Subject: [PATCH 007/144] [CU - #1ykyu7k] [CU - #1ykyu7k] feat: added test settings file --- .gitignore | 3 +- explorer/tests/test_views.py | 2 +- test_settings.py | 101 +++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 test_settings.py diff --git a/.gitignore b/.gitignore index b1d8cacc..53598004 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,10 @@ /dist *.egg-info .DS_Store +tmp /build *# *~ .coverage /htmlcov/ -*.orig \ No newline at end of file +*.orig diff --git a/explorer/tests/test_views.py b/explorer/tests/test_views.py index e93b788e..6f1eb34d 100644 --- a/explorer/tests/test_views.py +++ b/explorer/tests/test_views.py @@ -4,7 +4,7 @@ from django.conf import settings from django.forms.models import model_to_dict from explorer.tests.factories import SimpleQueryFactory, QueryLogFactory -from explorer.models import Query, QueryLog, MSG_FAILED_BLACKLIST +from explorer.models import Query, QueryLog, QueryChangeLog, MSG_FAILED_BLACKLIST from explorer.views import user_can_see_query from explorer.app_settings import EXPLORER_TOKEN from mock import Mock, patch diff --git a/test_settings.py b/test_settings.py new file mode 100644 index 00000000..665604b5 --- /dev/null +++ b/test_settings.py @@ -0,0 +1,101 @@ +import os +# import djcelery + +SECRET_KEY = 'shhh' +DEBUG = True +STATIC_URL = '/static/' + +ALLOWED_HOSTS = ['0.0.0.0', 'localhost', '127.0.0.1'] + +BASE_DIR = os.path.dirname(os.path.dirname(__file__)) + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'tmp', + 'TEST': { + 'NAME': 'test_tmp' + } + }, + 'alt': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'tmp2', + 'TEST': { + 'NAME': 'test_tmp2' + } + }, + 'not_registered': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'tmp3', + 'TEST': { + 'NAME': 'test_tmp3' + } + } +} + +EXPLORER_CONNECTIONS = { + #'Postgres': 'postgres', + #'MySQL': 'mysql', + 'SQLite': 'default', + 'Another': 'alt' +} +EXPLORER_DEFAULT_CONNECTION = 'default' + +ROOT_URLCONF = 'explorer.tests.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django.template.context_processors.static', + 'django.template.context_processors.request', + ], + 'debug': DEBUG + }, + }, +] + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.admin', + 'explorer', + # 'djcelery' +) + +AUTHENTICATION_BACKENDS = ( + "django.contrib.auth.backends.ModelBackend", +) + +MIDDLEWARE_CLASSES = [ + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', +] + +# TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner' + +# djcelery.setup_loader() +CELERY_ALWAYS_EAGER = True +BROKER_BACKEND = 'memory' + +# Explorer-specific + +EXPLORER_TRANSFORMS = ( + ('foo', '{0}'), + ('bar', 'x: {0}') +) + +EXPLORER_USER_QUERY_VIEWS = {} +EXPLORER_TASKS_ENABLED = True +EXPLORER_S3_BUCKET = 'thisismybucket.therearemanylikeit.butthisoneismine' From e7ccbf60ac3fa3ded96e8b72531406445d658573 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Fri, 25 Mar 2022 16:46:23 +0530 Subject: [PATCH 008/144] [CU - #1ykyu7k] refactor: code clean-up --- explorer/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/explorer/views.py b/explorer/views.py index c74914cc..5e715c56 100644 --- a/explorer/views.py +++ b/explorer/views.py @@ -319,7 +319,6 @@ def post(self, request, query_id): query, form = QueryView.get_instance_and_form(request, query_id) old_sql = query.sql; - # success = form.is_valid() and form.save() form_isvalid = form.is_valid() if form_isvalid: new_sql = request.POST.get('sql') From bda12515578543c375cd43ed418245b87c88f9c1 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Thu, 21 Apr 2022 22:57:08 +0530 Subject: [PATCH 009/144] [CU - ] feat: added new functions for checking replication lag --- explorer/utils.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/explorer/utils.py b/explorer/utils.py index 4d998410..0ec4dfc2 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -1,5 +1,3 @@ - - import functools import sys PY3 = sys.version_info[0] == 3 @@ -15,6 +13,7 @@ from django.http import HttpResponse from six.moves import cStringIO import sqlparse +import datetime EXPLORER_PARAM_TOKEN = "$$" @@ -238,9 +237,45 @@ def get_s3_connection(): app_settings.S3_SECRET_KEY, default_bucket=app_settings.S3_BUCKET) + def compare_sql(old_sql, new_sql): """ Compares whether two sql queries are the same after formatting them """ return fmt_sql(old_sql) == fmt_sql(new_sql) + + +def get_duration(td): + """ + Show time intervals (timedelta) in hours, minutes and seconds + """ + total_seconds = int(td.total_seconds()) + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + seconds = total_seconds - (hours * 3600) - (minutes * 60) + + if hours == 0: + return '{} minutes and {} seconds'.format(minutes, seconds) + + return '{} hours, {} minutes and {} seconds'.format(hours, minutes, seconds) + + +def check_replication_lag(): + """ + Check if a replication lag exists + :returns: True and the replication lag interval if it + exceeds 3 minutes, else returns False and None + """ + conn = get_connection() + cursor = conn.cursor() + + cursor.execute("SELECT now() - pg_last_xact_replay_timestamp() AS replication_lag") + replication_lag = cursor.fetchone()[0] + + threshold_value = datetime.timedelta(minutes=3) + + if not replication_lag or replication_lag <= threshold_value: + return False, None + + return True, get_duration(replication_lag) From 5a72a6964cf0ad0602283ef21ab32eeafd4c87eb Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Thu, 21 Apr 2022 22:58:54 +0530 Subject: [PATCH 010/144] [CU - ] refactor: modified views and templates to include replication lag check --- explorer/templates/explorer/play.html | 5 +++++ explorer/templates/explorer/query.html | 3 +++ explorer/views.py | 14 ++++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 0a337791..b195586c 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -1,5 +1,7 @@ {% extends "explorer/base.html" %} +{% load explorer_extras %} + {% block sql_explorer_navlinks %} {% if can_change %}
  • New Query
    • @@ -14,6 +16,9 @@

    Playground

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved.

    + {% if lag_exists %} +
    Warning! Some queries might be outdated by approximately {{ replication_lag }}.
    + {% endif %}
    {% csrf_token %} {% if error %}
    {{ error|escape }}
    diff --git a/explorer/templates/explorer/query.html b/explorer/templates/explorer/query.html index d9a632aa..e0f8a54f 100644 --- a/explorer/templates/explorer/query.html +++ b/explorer/templates/explorer/query.html @@ -18,6 +18,9 @@

    {% if query %}{{ query.title }}{% if shared %}  shared{{ message }}

    {% endif %} + {% if lag_exists %} +
    Warning! Some queries might be outdated by approximately {{ replication_lag }} mins
    + {% endif %}
    {% if query %} {% csrf_token %} diff --git a/explorer/views.py b/explorer/views.py index 5e715c56..1a35d9b3 100644 --- a/explorer/views.py +++ b/explorer/views.py @@ -30,7 +30,9 @@ user_can_see_query,\ fmt_sql,\ allowed_query_pks,\ - url_get_show, compare_sql + url_get_show,\ + compare_sql,\ + check_replication_lag try: from collections import Counter @@ -346,9 +348,13 @@ def query_viewmodel(request, query, title=None, form=None, message=None, run_que rows = url_get_rows(request) res = None ql = None + lag_exists = False + replication_lag = None if run_query: try: res, ql = query.execute_with_logging(request.user) + lag_exists, replication_lag = check_replication_lag() + print "Running new query" except DatabaseError as e: error = str(e) has_valid_results = not error and res and run_query @@ -370,5 +376,9 @@ def query_viewmodel(request, query, title=None, form=None, message=None, run_que 'dataUrl': reverse_lazy('query_csv', kwargs={'query_id': query.id}) if query.id else '', 'bucket': app_settings.S3_BUCKET, 'snapshots': query.snapshots if query.snapshot else [], - 'ql_id': ql.id if ql else None}) + 'ql_id': ql.id if ql else None, + 'lag_exists': lag_exists, + 'replication_lag': replication_lag, + } + ) return ret From 6b2eebef4cfde8ab12bacd4cbe7d6f585de1b35b Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Thu, 21 Apr 2022 23:01:09 +0530 Subject: [PATCH 011/144] [CU - ] refactor: code clean-up --- explorer/templates/explorer/play.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index b195586c..2f11ee75 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -1,7 +1,5 @@ {% extends "explorer/base.html" %} -{% load explorer_extras %} - {% block sql_explorer_navlinks %} {% if can_change %}
  • New Query
    • From ff6bec6f53ebe81c93faa9fa07126f994e1bfb11 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Thu, 21 Apr 2022 23:28:36 +0530 Subject: [PATCH 012/144] [CU - ] refactor: code clean-up --- explorer/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/explorer/views.py b/explorer/views.py index 1a35d9b3..e5f18fbd 100644 --- a/explorer/views.py +++ b/explorer/views.py @@ -354,7 +354,6 @@ def query_viewmodel(request, query, title=None, form=None, message=None, run_que try: res, ql = query.execute_with_logging(request.user) lag_exists, replication_lag = check_replication_lag() - print "Running new query" except DatabaseError as e: error = str(e) has_valid_results = not error and res and run_query From 08c8189cb5eb71caa497870d378da3c63fce312e Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Sun, 24 Apr 2022 16:56:36 +0530 Subject: [PATCH 013/144] [CU - ] feat: use 'ago' package to humanize timedelta --- explorer/templates/explorer/play.html | 2 +- explorer/templates/explorer/query.html | 2 +- explorer/utils.py | 18 ++---------------- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 2f11ee75..5c845797 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -15,7 +15,7 @@

    Playground

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved.

    {% if lag_exists %} -
    Warning! Some queries might be outdated by approximately {{ replication_lag }}.
    +
    Warning! The queryset results might be outdated since some data were last updated around {{ replication_lag }}.
    {% endif %} {% csrf_token %} {% if error %} diff --git a/explorer/templates/explorer/query.html b/explorer/templates/explorer/query.html index e0f8a54f..34cdc041 100644 --- a/explorer/templates/explorer/query.html +++ b/explorer/templates/explorer/query.html @@ -19,7 +19,7 @@

    {% if query %}{{ query.title }}{% if shared %}  shared{{ message }}

    {% endif %} {% if lag_exists %} -
    Warning! Some queries might be outdated by approximately {{ replication_lag }} mins
    +
    Warning! The queryset results might be outdated since some data were last updated around {{ replication_lag }}.
    {% endif %}
    {% if query %} diff --git a/explorer/utils.py b/explorer/utils.py index 0ec4dfc2..771ec546 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -12,6 +12,7 @@ from django.db import connections, connection, DatabaseError from django.http import HttpResponse from six.moves import cStringIO +from ago import human import sqlparse import datetime @@ -246,21 +247,6 @@ def compare_sql(old_sql, new_sql): return fmt_sql(old_sql) == fmt_sql(new_sql) -def get_duration(td): - """ - Show time intervals (timedelta) in hours, minutes and seconds - """ - total_seconds = int(td.total_seconds()) - hours = total_seconds // 3600 - minutes = (total_seconds % 3600) // 60 - seconds = total_seconds - (hours * 3600) - (minutes * 60) - - if hours == 0: - return '{} minutes and {} seconds'.format(minutes, seconds) - - return '{} hours, {} minutes and {} seconds'.format(hours, minutes, seconds) - - def check_replication_lag(): """ Check if a replication lag exists @@ -278,4 +264,4 @@ def check_replication_lag(): if not replication_lag or replication_lag <= threshold_value: return False, None - return True, get_duration(replication_lag) + return True, human(replication_lag, 4) From 5d583ff4fb84577ae7c38f233dd2cae46356c07c Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Sun, 24 Apr 2022 17:06:30 +0530 Subject: [PATCH 014/144] [CU - ] feat: added package 'ago' to requirements and set-up files --- requirements.txt | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b4f0851b..d952625d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ factory-boy==2.6.0 mock==1.0.1 six==1.10.0 sqlparse==0.1.18 -unicodecsv==0.14.1 \ No newline at end of file +unicodecsv==0.14.1 +ago==0.0.93 diff --git a/setup.py b/setup.py index 70dbb6cd..77a4114c 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ def read(fname): 'sqlparse>=0.1.11', 'unicodecsv>=0.13.0', 'six>=1.10.0', + 'ago>=0.0.93', ], include_package_data=True, zip_safe = False, From d11aa77bd3dc8b38de44c50c432305f9199b1d31 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Mon, 25 Apr 2022 12:11:51 +0530 Subject: [PATCH 015/144] [CU - ] refactor: change threshold value as a constant --- explorer/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/explorer/utils.py b/explorer/utils.py index 771ec546..f201c6e6 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -18,6 +18,8 @@ EXPLORER_PARAM_TOKEN = "$$" +REPLICATION_LAG_THRESHOLD_VALUE_IN_MINUTES = 3 + # SQL Specific Things @@ -259,7 +261,7 @@ def check_replication_lag(): cursor.execute("SELECT now() - pg_last_xact_replay_timestamp() AS replication_lag") replication_lag = cursor.fetchone()[0] - threshold_value = datetime.timedelta(minutes=3) + threshold_value = datetime.timedelta(minutes=REPLICATION_LAG_THRESHOLD_VALUE_IN_MINUTES) if not replication_lag or replication_lag <= threshold_value: return False, None From 208ee6eb4a48d7bc31396b0562eb152883651bba Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Mon, 25 Apr 2022 12:12:22 +0530 Subject: [PATCH 016/144] [CU - ] refactor: update warning message for replication lag --- explorer/templates/explorer/play.html | 2 +- explorer/templates/explorer/query.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 5c845797..5ba89a0a 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -15,7 +15,7 @@

    Playground

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved.

    {% if lag_exists %} -
    Warning! The queryset results might be outdated since some data were last updated around {{ replication_lag }}.
    +
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    {% endif %} {% csrf_token %} {% if error %} diff --git a/explorer/templates/explorer/query.html b/explorer/templates/explorer/query.html index 34cdc041..2be1b2ef 100644 --- a/explorer/templates/explorer/query.html +++ b/explorer/templates/explorer/query.html @@ -19,7 +19,7 @@

    {% if query %}{{ query.title }}{% if shared %}  shared{{ message }}

    {% endif %} {% if lag_exists %} -
    Warning! The queryset results might be outdated since some data were last updated around {{ replication_lag }}.
    +
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    {% endif %}
    {% if query %} From 46d1314db5b499fcae9336bea61f32fabd414495 Mon Sep 17 00:00:00 2001 From: Kartik Sharma Date: Wed, 4 May 2022 16:06:47 +0530 Subject: [PATCH 017/144] fixes py2/3 incompatibility of Query model --- explorer/models.py | 7 ++++--- requirements.txt | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/explorer/models.py b/explorer/models.py index d2b8b962..34a2d6d8 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -1,4 +1,5 @@ from explorer.utils import passes_blacklist, swap_params, extract_params, shared_dict_update, get_connection, get_s3_connection +from future.utils import python_2_unicode_compatible from django.db import models, DatabaseError from time import time from django.core.urlresolvers import reverse @@ -12,7 +13,7 @@ logger = logging.getLogger(__name__) - +@python_2_unicode_compatible class Query(models.Model): title = models.CharField(max_length=255) sql = models.TextField() @@ -31,8 +32,8 @@ class Meta: ordering = ['title'] verbose_name_plural = 'Queries' - def __unicode__(self): - return six.text_type(self.title) + def __str__(self): + return self.title def get_run_count(self): return self.querylog_set.count() diff --git a/requirements.txt b/requirements.txt index d952625d..e072b079 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ six==1.10.0 sqlparse==0.1.18 unicodecsv==0.14.1 ago==0.0.93 +# py2/3 compatible libraries +future==0.18.2 From dbd8e6132f49e3acb562dcb1356c4a8b54c1b196 Mon Sep 17 00:00:00 2001 From: Kartik Sharma Date: Wed, 4 May 2022 18:25:28 +0530 Subject: [PATCH 018/144] fixes deps not getting installed on package user's repo --- requirements.txt | 3 --- setup.py | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index e072b079..87dba986 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,3 @@ mock==1.0.1 six==1.10.0 sqlparse==0.1.18 unicodecsv==0.14.1 -ago==0.0.93 -# py2/3 compatible libraries -future==0.18.2 diff --git a/setup.py b/setup.py index 77a4114c..ef963188 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,9 @@ def read(fname): 'sqlparse>=0.1.11', 'unicodecsv>=0.13.0', 'six>=1.10.0', + # custom deps 'ago>=0.0.93', + 'future==0.18.2' # py2/3 compatible libraries ], include_package_data=True, zip_safe = False, From 0097a86c5ad4e206da01b7f790cb99cd75a291e7 Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Fri, 27 May 2022 10:36:35 +0530 Subject: [PATCH 019/144] integrate clarity for explorer --- explorer/templates/explorer/base.html | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index 392d0a85..7866576c 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -1,6 +1,5 @@ {% load staticfiles %} - - + @@ -22,6 +21,13 @@ dataUrl = "{{ dataUrl }}"; queryId = "{% firstof query.id 'new' %}"; + From 9309fd5c36a2d36b93da27729ebc081ca9115bca Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Fri, 27 May 2022 10:48:54 +0530 Subject: [PATCH 020/144] [CU - #2a2abq2] clean up --- explorer/templates/explorer/base.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index 7866576c..5e6aa86d 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -1,5 +1,6 @@ {% load staticfiles %} - + + From 91533b12de1b44e96bd6976617f3dc0613ab0a90 Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Mon, 30 May 2022 19:42:48 +0530 Subject: [PATCH 021/144] [CU - #2a2abq2] add custom filters --- explorer/templates/explorer/base.html | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index 5e6aa86d..da5554f2 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -22,13 +22,16 @@ dataUrl = "{{ dataUrl }}"; queryId = "{% firstof query.id 'new' %}"; - + {% block sql_explorer_content %} + + {% endblock %} From efa912cef6dc608332a7de264bd63300786c03d8 Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Mon, 30 May 2022 19:46:07 +0530 Subject: [PATCH 022/144] [CU - #2a2abq2] refactor --- explorer/templates/explorer/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index da5554f2..6e887529 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -29,7 +29,7 @@ t=l.createElement(r);t.async=1;t.src="https://www.clarity.ms/tag/"+i; y=l.getElementsByTagName(r)[0];y.parentNode.insertBefore(t,y); })(window, document, "clarity", "script", "c1c6rvg99w"); - clarity("set", "email_id", "{{ object.run_by_user.email }}") + clarity("set", "email_id", "{{ object.run_by_user.email }}"); {% endblock %} From 76cb1eea9125785b6bf88dd2b5401ff544e00afe Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Tue, 31 May 2022 16:01:03 +0530 Subject: [PATCH 023/144] [CU - #2a2abq2] remove duplicate block names --- explorer/templates/explorer/base.html | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index 6e887529..9a3ae17d 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -22,16 +22,14 @@ dataUrl = "{{ dataUrl }}"; queryId = "{% firstof query.id 'new' %}"; - {% block sql_explorer_content %} - - {% endblock %} + From 123f1dbf4f31c5ba6ef845e8d4e968bcc09fa987 Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Tue, 31 May 2022 17:09:33 +0530 Subject: [PATCH 024/144] [CU - #2a2abq2] get user from request --- explorer/templates/explorer/base.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index 9a3ae17d..f678be62 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -28,7 +28,8 @@ t=l.createElement(r);t.async=1;t.src="https://www.clarity.ms/tag/"+i; y=l.getElementsByTagName(r)[0];y.parentNode.insertBefore(t,y); })(window, document, "clarity", "script", "c1c6rvg99w"); - clarity("set", "email_id", "{{ user.email }}"); + clarity("set", "user_first_name", '{{ request.user.get_short_name }}') + clarity("set", "email_id", '{{ request.user.email }}'); From eddcc244936a4698a34ba53f8e1f29952f2f0e8f Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Tue, 31 May 2022 17:18:04 +0530 Subject: [PATCH 025/144] [CU - #2a2abq2] get full name instead of short name --- explorer/templates/explorer/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index f678be62..eef7f998 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -28,7 +28,7 @@ t=l.createElement(r);t.async=1;t.src="https://www.clarity.ms/tag/"+i; y=l.getElementsByTagName(r)[0];y.parentNode.insertBefore(t,y); })(window, document, "clarity", "script", "c1c6rvg99w"); - clarity("set", "user_first_name", '{{ request.user.get_short_name }}') + clarity("set", "user_first_name", '{{ request.user.get_full_name }}') clarity("set", "email_id", '{{ request.user.email }}'); From 7fa74e7d870b1758f300bcd2c49712c9617ea6c5 Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Tue, 31 May 2022 18:16:34 +0530 Subject: [PATCH 026/144] [CU - #2a2abq2] refactor --- explorer/templates/explorer/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index eef7f998..b0d0c744 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -28,7 +28,7 @@ t=l.createElement(r);t.async=1;t.src="https://www.clarity.ms/tag/"+i; y=l.getElementsByTagName(r)[0];y.parentNode.insertBefore(t,y); })(window, document, "clarity", "script", "c1c6rvg99w"); - clarity("set", "user_first_name", '{{ request.user.get_full_name }}') + clarity("set", "user_full_name", '{{ request.user.get_full_name }}') clarity("set", "email_id", '{{ request.user.email }}'); From 41a264ea36efe94d7ef3c16bc8f5cd0c11f24590 Mon Sep 17 00:00:00 2001 From: Srujana Manvi Date: Wed, 1 Jun 2022 16:16:29 +0530 Subject: [PATCH 027/144] [CU - #2a2abq2] add username filter --- explorer/templates/explorer/base.html | 1 + 1 file changed, 1 insertion(+) diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index b0d0c744..8cd31b1d 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -29,6 +29,7 @@ y=l.getElementsByTagName(r)[0];y.parentNode.insertBefore(t,y); })(window, document, "clarity", "script", "c1c6rvg99w"); clarity("set", "user_full_name", '{{ request.user.get_full_name }}') + clarity("set", "username", '{{ request.user.get_username }}') clarity("set", "email_id", '{{ request.user.email }}'); From 53ec2cdebb343a9543f16e398dc6c4e0bb49b7e3 Mon Sep 17 00:00:00 2001 From: Shravan A J Date: Thu, 7 Jul 2022 12:34:01 +0530 Subject: [PATCH 028/144] [CU - ] refactor: change ago dependency to 0.0.93 exact match --- requirements.txt | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87dba986..d952625d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ mock==1.0.1 six==1.10.0 sqlparse==0.1.18 unicodecsv==0.14.1 +ago==0.0.93 diff --git a/setup.py b/setup.py index ef963188..35cf82a4 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ def read(fname): 'unicodecsv>=0.13.0', 'six>=1.10.0', # custom deps - 'ago>=0.0.93', + 'ago==0.0.93', 'future==0.18.2' # py2/3 compatible libraries ], include_package_data=True, From d2019e8871aa83b97ba8a3801423ffee449dda9e Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Tue, 7 Feb 2023 12:37:41 +0530 Subject: [PATCH 029/144] added new migration 0006 snapshot --- .../migrations/0006_auto_20230207_0103.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 explorer/migrations/0006_auto_20230207_0103.py diff --git a/explorer/migrations/0006_auto_20230207_0103.py b/explorer/migrations/0006_auto_20230207_0103.py new file mode 100644 index 00000000..a9f2d141 --- /dev/null +++ b/explorer/migrations/0006_auto_20230207_0103.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('explorer', '0005_querychangelog'), + ] + + operations = [ + migrations.AlterField( + model_name='query', + name='snapshot', + field=models.BooleanField(default=False, help_text='Include in snapshot task (if enabled)'), + ), + ] From 778ba820ace194c08ab41bfc3231f472e388ba57 Mon Sep 17 00:00:00 2001 From: Ujjwal-Squadstack <73160338+Ujjwal-Squadstack@users.noreply.github.com> Date: Wed, 15 Mar 2023 17:10:38 +0530 Subject: [PATCH 030/144] add info alert for revoking column access --- explorer/templates/explorer/play.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 5ba89a0a..edc6b533 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -14,6 +14,11 @@

    Playground

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved.

    +
    + Note: Starting March 16th, access to the phone_number column has been removed for the following + tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, and + voice_compliance_businessoutreach. Please avoid using `select *` for these tables. +
    {% if lag_exists %}
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    {% endif %} From 378a3395558157d7eb2f4479cb27e6e4917ba902 Mon Sep 17 00:00:00 2001 From: Ujjwal-Squadstack <73160338+Ujjwal-Squadstack@users.noreply.github.com> Date: Thu, 16 Mar 2023 10:15:07 +0530 Subject: [PATCH 031/144] add info alert for revoking column access --- explorer/templates/explorer/play.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index edc6b533..92d125da 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -16,7 +16,8 @@

    Playground

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved.

    Note: Starting March 16th, access to the phone_number column has been removed for the following - tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, and + tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, + whatsapp2_whatsapp2inboundmessage, whatsapp2_whatsapp2outboundmessage and voice_compliance_businessoutreach. Please avoid using `select *` for these tables.
    {% if lag_exists %} From 2a19ee3bfe7336fe64ee31353f59683707a8d59e Mon Sep 17 00:00:00 2001 From: Ujjwal-Squadstack <73160338+Ujjwal-Squadstack@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:04:28 +0530 Subject: [PATCH 032/144] add warning alert --- explorer/templates/explorer/play.html | 10 +++++----- explorer/templates/explorer/query.html | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 92d125da..aa84b76d 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -14,15 +14,15 @@

    Playground

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved.

    -
    + {% if lag_exists %} +
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    + {% endif %} +
    Note: Starting March 16th, access to the phone_number column has been removed for the following tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, whatsapp2_whatsapp2inboundmessage, whatsapp2_whatsapp2outboundmessage and - voice_compliance_businessoutreach. Please avoid using `select *` for these tables. + voice_compliance_businessoutreach. Please avoid using select * for these tables.
    - {% if lag_exists %} -
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    - {% endif %} {% csrf_token %} {% if error %}
    {{ error|escape }}
    diff --git a/explorer/templates/explorer/query.html b/explorer/templates/explorer/query.html index 2be1b2ef..c1455aa2 100644 --- a/explorer/templates/explorer/query.html +++ b/explorer/templates/explorer/query.html @@ -18,6 +18,12 @@

    {% if query %}{{ query.title }}{% if shared %}  shared{{ message }}

    {% endif %} +
    + Note: Starting March 16th, access to the phone_number column has been removed for the following + tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, + whatsapp2_whatsapp2inboundmessage, whatsapp2_whatsapp2outboundmessage and + voice_compliance_businessoutreach. Please avoid using `select *` for these tables. +
    {% if lag_exists %}
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    {% endif %} From b086fcb8e68f6ab36a111c97e3ef18d335b019d0 Mon Sep 17 00:00:00 2001 From: Ujjwal-Squadstack <73160338+Ujjwal-Squadstack@users.noreply.github.com> Date: Thu, 16 Mar 2023 16:31:00 +0530 Subject: [PATCH 033/144] add paragraph --- explorer/templates/explorer/play.html | 9 +++++---- explorer/templates/explorer/query.html | 6 ------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index aa84b76d..c72e3abb 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -17,12 +17,13 @@

    Playground

    {% if lag_exists %}
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    {% endif %} -
    - Note: Starting March 16th, access to the phone_number column has been removed for the following +
    +

    + Note: Starting March 16th, access to the phone_number column has been removed for the following tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, whatsapp2_whatsapp2inboundmessage, whatsapp2_whatsapp2outboundmessage and - voice_compliance_businessoutreach. Please avoid using select * for these tables. -

    + voice_compliance_businessoutreach. Please avoid using `select *` for these tables. +

    {% csrf_token %} {% if error %}
    {{ error|escape }}
    diff --git a/explorer/templates/explorer/query.html b/explorer/templates/explorer/query.html index c1455aa2..2be1b2ef 100644 --- a/explorer/templates/explorer/query.html +++ b/explorer/templates/explorer/query.html @@ -18,12 +18,6 @@

    {% if query %}{{ query.title }}{% if shared %}  shared{{ message }}

    {% endif %} -
    - Note: Starting March 16th, access to the phone_number column has been removed for the following - tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, - whatsapp2_whatsapp2inboundmessage, whatsapp2_whatsapp2outboundmessage and - voice_compliance_businessoutreach. Please avoid using `select *` for these tables. -
    {% if lag_exists %}
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    {% endif %} From 54d476b31cb26baabd785a0cf22bd29cb981f288 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Wed, 5 Apr 2023 20:52:26 +0530 Subject: [PATCH 034/144] new connection added --- explorer/app_settings.py | 1 + explorer/models.py | 23 ++++++++++++++++++----- explorer/utils.py | 3 +++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/explorer/app_settings.py b/explorer/app_settings.py index 004bb058..2104f8b6 100644 --- a/explorer/app_settings.py +++ b/explorer/app_settings.py @@ -2,6 +2,7 @@ # Required EXPLORER_CONNECTION_NAME = getattr(settings, 'EXPLORER_CONNECTION_NAME', None) +EXPLORER_CONNECTION_PII_NAME = getattr(settings, 'EXPLORER_CONNECTION_PII_NAME', None) # Change the behavior of explorer EXPLORER_SQL_BLACKLIST = getattr(settings, 'EXPLORER_SQL_BLACKLIST', ('ALTER', 'RENAME ', 'DROP', 'TRUNCATE', 'INSERT INTO', 'UPDATE', 'REPLACE', 'DELETE', 'CREATE TABLE', 'SCHEMA', 'GRANT', 'OWNER TO')) diff --git a/explorer/models.py b/explorer/models.py index 34a2d6d8..198bb32b 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -1,4 +1,4 @@ -from explorer.utils import passes_blacklist, swap_params, extract_params, shared_dict_update, get_connection, get_s3_connection +from explorer.utils import passes_blacklist, swap_params, extract_params, shared_dict_update, get_connection, get_s3_connection , get_connection_pii from future.utils import python_2_unicode_compatible from django.db import models, DatabaseError from time import time @@ -22,6 +22,7 @@ class Query(models.Model): created_at = models.DateTimeField(auto_now_add=True) last_run_date = models.DateTimeField(auto_now=True) snapshot = models.BooleanField(default=False, help_text="Include in snapshot task (if enabled)") + # connection=models.BooleanField(default=False, help_text="use to select connection type") def __init__(self, *args, **kwargs): self.params = kwargs.get('params') @@ -47,8 +48,9 @@ def passes_blacklist(self): def final_sql(self): return swap_params(self.sql, self.available_params()) - def execute_query_only(self): - return QueryResult(self.final_sql()) + def execute_query_only(self,connection_type=None): + + return QueryResult(self.final_sql(),connection_type) def execute_with_logging(self, executing_user): ql = self.log(executing_user) @@ -61,6 +63,13 @@ def execute(self): ret = self.execute_query_only() ret.process() return ret + + def execute_pii(self): + ret = self.execute_query_only(True) + ret.process() + return ret + + def available_params(self): """ @@ -131,7 +140,7 @@ class Meta: class QueryResult(object): - def __init__(self, sql): + def __init__(self, sql, connection_type=None): self.sql = sql @@ -140,6 +149,7 @@ def __init__(self, sql): self._description = cursor.description or [] self._data = [list(r) for r in cursor.fetchall()] self.duration = duration + self.connection_type=connection_type cursor.close() @@ -193,7 +203,10 @@ def process_rows(self): r[ix] = t.format(str(r[ix])) def execute_query(self): - conn = get_connection() + if(self.connection_type): + conn = get_connection_pii() # can change connectiion type here to use different role --> get_connection_pii() + else: + conn= get_connection() cursor = conn.cursor() start_time = time() diff --git a/explorer/utils.py b/explorer/utils.py index f201c6e6..a8322748 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -32,6 +32,9 @@ def passes_blacklist(sql): def get_connection(): return connections[app_settings.EXPLORER_CONNECTION_NAME] if app_settings.EXPLORER_CONNECTION_NAME else connection +def get_connection_pii(): + return connections[app_settings.EXPLORER_CONNECTION_NAME_PII] if app_settings.EXPLORER_CONNECTION_NAME_PII else connection + def schema_info(): """ From f88756e505339b562b55a1846394811c2b12505c Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Wed, 5 Apr 2023 23:53:38 +0530 Subject: [PATCH 035/144] debug --- explorer/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/explorer/models.py b/explorer/models.py index 198bb32b..4e80b5ad 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -22,7 +22,7 @@ class Query(models.Model): created_at = models.DateTimeField(auto_now_add=True) last_run_date = models.DateTimeField(auto_now=True) snapshot = models.BooleanField(default=False, help_text="Include in snapshot task (if enabled)") - # connection=models.BooleanField(default=False, help_text="use to select connection type") + connection=models.BooleanField(default=False, help_text="use to select connection type") def __init__(self, *args, **kwargs): self.params = kwargs.get('params') @@ -150,6 +150,8 @@ def __init__(self, sql, connection_type=None): self._data = [list(r) for r in cursor.fetchall()] self.duration = duration self.connection_type=connection_type + print("----------------------------debug-------------------------") + print(self) cursor.close() From dea5296911f9e0b28eac3c194f7a4c699bc337e2 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Thu, 6 Apr 2023 00:00:10 +0530 Subject: [PATCH 036/144] debug --- explorer/models.py | 54 +++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/explorer/models.py b/explorer/models.py index 4e80b5ad..3622a46c 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -1,4 +1,4 @@ -from explorer.utils import passes_blacklist, swap_params, extract_params, shared_dict_update, get_connection, get_s3_connection , get_connection_pii +from explorer.utils import passes_blacklist, swap_params, extract_params, shared_dict_update, get_connection, get_s3_connection, get_connection_pii from future.utils import python_2_unicode_compatible from django.db import models, DatabaseError from time import time @@ -13,16 +13,20 @@ logger = logging.getLogger(__name__) + @python_2_unicode_compatible class Query(models.Model): title = models.CharField(max_length=255) sql = models.TextField() description = models.TextField(null=True, blank=True) - created_by_user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True) + created_by_user = models.ForeignKey( + settings.AUTH_USER_MODEL, null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) last_run_date = models.DateTimeField(auto_now=True) - snapshot = models.BooleanField(default=False, help_text="Include in snapshot task (if enabled)") - connection=models.BooleanField(default=False, help_text="use to select connection type") + snapshot = models.BooleanField( + default=False, help_text="Include in snapshot task (if enabled)") + connection = models.BooleanField( + default=False, help_text="use to select connection type") def __init__(self, *args, **kwargs): self.params = kwargs.get('params') @@ -48,9 +52,9 @@ def passes_blacklist(self): def final_sql(self): return swap_params(self.sql, self.available_params()) - def execute_query_only(self,connection_type=None): - - return QueryResult(self.final_sql(),connection_type) + def execute_query_only(self, connection_type=None): + + return QueryResult(self.final_sql(), connection_type) def execute_with_logging(self, executing_user): ql = self.log(executing_user) @@ -63,13 +67,11 @@ def execute(self): ret = self.execute_query_only() ret.process() return ret - + def execute_pii(self): ret = self.execute_query_only(True) ret.process() return ret - - def available_params(self): """ @@ -109,8 +111,10 @@ def snapshots(self): class QueryLog(models.Model): sql = models.TextField(null=True, blank=True) - query = models.ForeignKey(Query, null=True, blank=True, on_delete=models.SET_NULL) - run_by_user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True) + query = models.ForeignKey( + Query, null=True, blank=True, on_delete=models.SET_NULL) + run_by_user = models.ForeignKey( + settings.AUTH_USER_MODEL, null=True, blank=True) run_at = models.DateTimeField(auto_now_add=True) duration = models.FloatField(blank=True, null=True) # milliseconds @@ -126,8 +130,10 @@ class QueryChangeLog(models.Model): old_sql = models.TextField(null=True, blank=True) new_sql = models.TextField(null=True, blank=True) - query = models.ForeignKey(Query, null=True, blank=True, on_delete=models.SET_NULL) - run_by_user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True) + query = models.ForeignKey( + Query, null=True, blank=True, on_delete=models.SET_NULL) + run_by_user = models.ForeignKey( + settings.AUTH_USER_MODEL, null=True, blank=True) run_at = models.DateTimeField(auto_now_add=True) @property @@ -149,9 +155,9 @@ def __init__(self, sql, connection_type=None): self._description = cursor.description or [] self._data = [list(r) for r in cursor.fetchall()] self.duration = duration - self.connection_type=connection_type + self.connection_type = connection_type print("----------------------------debug-------------------------") - print(self) + print(self) cursor.close() @@ -191,7 +197,8 @@ def process(self): self.process_columns() self.process_rows() - logger.info("Explorer Query Processing took %sms." % ((time() - start_time) * 1000)) + logger.info("Explorer test Query Processing took %sms." % + ((time() - start_time) * 1000)) def process_columns(self): for ix in self._get_numerics(): @@ -205,10 +212,11 @@ def process_rows(self): r[ix] = t.format(str(r[ix])) def execute_query(self): - if(self.connection_type): - conn = get_connection_pii() # can change connectiion type here to use different role --> get_connection_pii() + if (self.connection_type): + # can change connectiion type here to use different role --> get_connection_pii() + conn = get_connection_pii() else: - conn= get_connection() + conn = get_connection() cursor = conn.cursor() start_time = time() @@ -246,7 +254,8 @@ def __init__(self, label, statfn, precision=2, handles_null=False): self.handles_null = handles_null def __call__(self, coldata): - self.value = round(float(self.statfn(coldata)), self.precision) if coldata else 0 + self.value = round(float(self.statfn(coldata)), + self.precision) if coldata else 0 def __unicode__(self): return self.label @@ -264,7 +273,8 @@ def __init__(self, header, col): ColumnStat("Avg", lambda x: float(sum(x)) / float(len(x))), ColumnStat("Min", min), ColumnStat("Max", max), - ColumnStat("NUL", lambda x: int(sum(map(lambda y: 1 if y is None else 0, x))), 0, True) + ColumnStat("NUL", lambda x: int( + sum(map(lambda y: 1 if y is None else 0, x))), 0, True) ] without_nulls = list(map(lambda x: 0 if x is None else x, col)) From 0130f5e070c4c68cd102b69434ad586cd9b32615 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Thu, 6 Apr 2023 00:23:08 +0530 Subject: [PATCH 037/144] debug --- explorer/templates/explorer/play.html | 68 ++++++++++++++------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index c72e3abb..32e4a259 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -1,45 +1,50 @@ {% extends "explorer/base.html" %} {% block sql_explorer_navlinks %} - {% if can_change %} -
  • New Query
    • -
  • Playground
    • -
  • Logs
    • -
  • Change Logs
    • - {% endif %} +{% if can_change %} +
  • New Query
    • +
  • Playground
    • +
  • Logs
    • +
  • Change Logs
    • +{% endif %} {% endblock %} {% block sql_explorer_content %}
    -

    Playground

    -

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be saved.

    +

    Playground Test

    +

    The playground is for experimenting and writing ad-hoc queries. By default, nothing you do here will be + saved.

    {% if lag_exists %} -
    Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
    +
    Warning! The query results might be outdated since some data was last updated + around {{ replication_lag }}.
    {% endif %}

    - Note: Starting March 16th, access to the phone_number column has been removed for the following + Note: Starting March 16th, access to the phone_number column has been removed for the + following tables: voice_workflow_campaigncallcontact, voice_core_callcontact, compliance_outreachblacklist, whatsapp2_whatsapp2inboundmessage, whatsapp2_whatsapp2outboundmessage and voice_compliance_businessoutreach. Please avoid using `select *` for these tables.

    - {% csrf_token %} + {% + csrf_token %} {% if error %} -
    {{ error|escape }}
    +
    {{ error|escape }}
    {% endif %}
    -
    -
    - Playground SQL -
    - {% if ql_id %} +
    +
    + Playground SQL +
    + {% if ql_id %}
    - +
    - {% endif %} -
    + {% endif %} +
    @@ -48,11 +53,12 @@

    Playground

    - - - + + + - +
    @@ -69,11 +75,9 @@

    Playground

    {% endblock %} {% block sql_explorer_scripts %} - -{% endblock %} - - + +{% endblock %} \ No newline at end of file From 4beb391043e89182d3da22810e0a2ef8fb845ef7 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Thu, 6 Apr 2023 23:30:49 +0530 Subject: [PATCH 038/144] debug --- explorer/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/explorer/models.py b/explorer/models.py index 3622a46c..0c50217c 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -155,7 +155,8 @@ def __init__(self, sql, connection_type=None): self._description = cursor.description or [] self._data = [list(r) for r in cursor.fetchall()] self.duration = duration - self.connection_type = connection_type + if(connection_type): + self.connection_type = connection_type print("----------------------------debug-------------------------") print(self) From b6b7d9414ff9564c1fe938f09dba35e4c6fed130 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Thu, 6 Apr 2023 23:36:19 +0530 Subject: [PATCH 039/144] debug --- explorer/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/explorer/models.py b/explorer/models.py index 0c50217c..4b9f8391 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -155,8 +155,10 @@ def __init__(self, sql, connection_type=None): self._description = cursor.description or [] self._data = [list(r) for r in cursor.fetchall()] self.duration = duration - if(connection_type): + if (connection_type): self.connection_type = connection_type + else: + self.connection_type = False print("----------------------------debug-------------------------") print(self) From c44c4fc934d2ff0c9133dc2fffadc908a135741c Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Fri, 7 Apr 2023 00:09:32 +0530 Subject: [PATCH 040/144] debug --- explorer/models.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/explorer/models.py b/explorer/models.py index 4b9f8391..1a55cd3d 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -64,7 +64,7 @@ def execute_with_logging(self, executing_user): return ret, ql def execute(self): - ret = self.execute_query_only() + ret = self.execute_query_only(False) ret.process() return ret @@ -149,18 +149,22 @@ class QueryResult(object): def __init__(self, sql, connection_type=None): self.sql = sql + logger.info( + "----------------------------debug1-------------------------") + if (connection_type): + self.connection_type = connection_type + else: + self.connection_type = False cursor, duration = self.execute_query() self._description = cursor.description or [] self._data = [list(r) for r in cursor.fetchall()] self.duration = duration - if (connection_type): - self.connection_type = connection_type - else: - self.connection_type = False - print("----------------------------debug-------------------------") - print(self) + + logger.info( + "----------------------------debug-------------------------") + logger.info(self) cursor.close() From cbbd881d3f41636dd60c5205705d90d0f437bb11 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Fri, 7 Apr 2023 15:12:43 +0530 Subject: [PATCH 041/144] bug fixed --- explorer/utils.py | 57 ++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/explorer/utils.py b/explorer/utils.py index a8322748..d518acd3 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -1,3 +1,15 @@ +from django.contrib.auth import REDIRECT_FIELD_NAME +from django.contrib.auth.views import login +from django.contrib.admin.forms import AdminAuthenticationForm +import datetime +import sqlparse +from ago import human +from six.moves import cStringIO +from django.http import HttpResponse +from django.db import connections, connection, DatabaseError +from explorer import app_settings +import string +import re import functools import sys PY3 = sys.version_info[0] == 3 @@ -6,15 +18,6 @@ import csv else: import unicodecsv as csv -import re -import string -from explorer import app_settings -from django.db import connections, connection, DatabaseError -from django.http import HttpResponse -from six.moves import cStringIO -from ago import human -import sqlparse -import datetime EXPLORER_PARAM_TOKEN = "$$" @@ -24,14 +27,18 @@ def passes_blacklist(sql): - clean = functools.reduce(lambda sql, term: sql.upper().replace(term, ""), [t.upper() for t in app_settings.EXPLORER_SQL_WHITELIST], sql) - fails = [bl_word for bl_word in app_settings.EXPLORER_SQL_BLACKLIST if bl_word in clean.upper()] + clean = functools.reduce(lambda sql, term: sql.upper().replace(term, ""), [ + t.upper() for t in app_settings.EXPLORER_SQL_WHITELIST], sql) + fails = [ + bl_word for bl_word in app_settings.EXPLORER_SQL_BLACKLIST if bl_word in clean.upper()] return not any(fails), fails def get_connection(): + print("-------------------explorer_connection---------------------") return connections[app_settings.EXPLORER_CONNECTION_NAME] if app_settings.EXPLORER_CONNECTION_NAME else connection + def get_connection_pii(): return connections[app_settings.EXPLORER_CONNECTION_NAME_PII] if app_settings.EXPLORER_CONNECTION_NAME_PII else connection @@ -59,20 +66,21 @@ def schema_info(): for label, app in apps.app_configs.items(): if app.name not in app_settings.EXPLORER_SCHEMA_EXCLUDE_APPS: for model_name, model in apps.get_app_config(label).models.items(): - friendly_model = "%s -> %s" % (app.name, model._meta.object_name) + friendly_model = "%s -> %s" % (app.name, + model._meta.object_name) ret.append(( - friendly_model, - model._meta.db_table, - [_format_field(f) for f in model._meta.fields] - )) + friendly_model, + model._meta.db_table, + [_format_field(f) for f in model._meta.fields] + )) # Do the same thing for many_to_many fields. These don't show up in the field list of the model # because they are stored as separate "through" relations and have their own tables ret += [( - friendly_model, - m2m.rel.through._meta.db_table, - [_format_field(f) for f in m2m.rel.through._meta.fields] - ) for m2m in model._meta.many_to_many] + friendly_model, + m2m.rel.through._meta.db_table, + [_format_field(f) for f in m2m.rel.through._meta.fields] + ) for m2m in model._meta.many_to_many] return sorted(ret, key=lambda t: t[1]) @@ -149,9 +157,6 @@ def csv_report(query, delim=None): # Helpers -from django.contrib.admin.forms import AdminAuthenticationForm -from django.contrib.auth.views import login -from django.contrib.auth import REDIRECT_FIELD_NAME def safe_admin_login_prompt(request): @@ -261,10 +266,12 @@ def check_replication_lag(): conn = get_connection() cursor = conn.cursor() - cursor.execute("SELECT now() - pg_last_xact_replay_timestamp() AS replication_lag") + cursor.execute( + "SELECT now() - pg_last_xact_replay_timestamp() AS replication_lag") replication_lag = cursor.fetchone()[0] - threshold_value = datetime.timedelta(minutes=REPLICATION_LAG_THRESHOLD_VALUE_IN_MINUTES) + threshold_value = datetime.timedelta( + minutes=REPLICATION_LAG_THRESHOLD_VALUE_IN_MINUTES) if not replication_lag or replication_lag <= threshold_value: return False, None From ecb62a69d66fc97da892839edaf9d15fb5714f33 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Fri, 7 Apr 2023 18:06:36 +0530 Subject: [PATCH 042/144] debug --- explorer/templates/explorer/query_list.html | 148 ++++++++++---------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/explorer/templates/explorer/query_list.html b/explorer/templates/explorer/query_list.html index 29775a71..bc747d8d 100644 --- a/explorer/templates/explorer/query_list.html +++ b/explorer/templates/explorer/query_list.html @@ -2,118 +2,120 @@ {% load staticfiles %} {% block sql_explorer_navlinks %} - {% if can_change %} -
  • New Query
    • -
  • Playground
    • -
  • Logs
    • -
  • Change Logs
    • - {% endif %} +{% if can_change %} +
  • New Query
    • +
  • Play
    • +
  • Logs
    • +
  • Change Logs
    • +{% endif %} {% endblock %} {% block sql_explorer_content %} - {% if recent_queries|length > 0 %} -

    {{ recent_queries|length }} Most Recently Used

    - - - - - - - - - - {% for object in recent_queries %} - - - - - - {% endfor %} - -
    QueryLast RunCSV
    {{ object.title }}{{ object.last_run_date|date:"SHORT_DATETIME_FORMAT" }} - -
    - {% endif %} +{% if recent_queries|length > 0 %} +

    {{ recent_queries|length }} Most Recently Used

    + + + + + + + + + + {% for object in recent_queries %} + + + + + + {% endfor %} + +
    QueryLast RunCSV
    {{ object.title }}{{ object.last_run_date|date:"SHORT_DATETIME_FORMAT" }} + +
    +{% endif %} -
    -
    +
    +
    -

    All Queries

    +

    All Queries

    - +
    -
    +
    {% if tasks_enabled %} - + {% endif %} {% if can_change %} - - + + {% endif %} {% for object in object_list %} - + {% if object.is_header %} - + + {% else %} - - - {% if tasks_enabled %} - - {% endif %} - - {% if can_change %} - - + + + + {% if tasks_enabled %} + + {% endif %} + + {% if can_change %} + + + {% endif %} + {% endif %} {% endfor %}
    Query CreatedEmailEmailCSVPlayDeletePlayDeleteRun Count
    - - + + + {{ object.title }} ({{ object.count }}) - - - {% if object.is_in_category %} → {% endif %}{{ object.title }} - {{ object.created_at|date:"SHORT_DATE_FORMAT" }} - {% if object.created_by_user %} - by {{ object.created_by_user }} - {% endif %} - - - - - - - - - + {% if object.is_in_category %} → {% endif %}{{ object.title }} + {{ object.created_at|date:"SHORT_DATE_FORMAT" }} + {% if object.created_by_user %} + by {{ object.created_by_user }} {% endif %} - {{ object.run_count }} + + + + + + + + {{ object.run_count }}
    -
    +
    {% endblock %} {% block sql_explorer_scripts %} - -{% endblock %} +{% endblock %} \ No newline at end of file From 4e043759469422ab08ce4e34813905372e2683da Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Fri, 7 Apr 2023 19:00:18 +0530 Subject: [PATCH 043/144] debug --- explorer/templates/explorer/play.html | 4 +- .../templates/explorer/querylog_list.html | 74 ++++++++++--------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 32e4a259..ba582845 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -2,8 +2,8 @@ {% block sql_explorer_navlinks %} {% if can_change %} -
  • New Query
    • -
  • Playground
    • + +
  • Play
  • Logs
  • Change Logs
    • {% endif %} diff --git a/explorer/templates/explorer/querylog_list.html b/explorer/templates/explorer/querylog_list.html index d40a4fa4..32418f2d 100644 --- a/explorer/templates/explorer/querylog_list.html +++ b/explorer/templates/explorer/querylog_list.html @@ -1,53 +1,55 @@ {% extends "explorer/base.html" %} {% block sql_explorer_navlinks %} - {% if can_change %} -
  • New Query
    • -
  • Playground
    • -
  • Logs
    • -
  • Change Logs
    • - {% endif %} +{% if can_change %} +
  • New Query
    • +
  • Play
    • +
  • Logs
    • +
  • Change Logs
    • +{% endif %} {% endblock %} {% block sql_explorer_content %} -

    Recent Query Logs - Page {{ page_obj.number }}

    - +

    Recent Query Logs - Page {{ page_obj.number }}

    +
    - - - - - - - {% for object in recent_logs %} - - - - - - - - - {% endfor %} - -
    Run At Run By DurationSQLQuery IDPlayground
    {{ object.run_at|date:"SHORT_DATETIME_FORMAT" }}{{ object.run_by_user.email }}{{ object.duration|floatformat:2 }}ms{{ object.sql }} {% if object.query_id %}Query {{ object.query_id }}{% elif object.is_playground %}Playground{% else %}--{% endif %}Open
    - {% if is_paginated %} - +{% endif %} +{% endblock %} \ No newline at end of file From ba05ba5cf97d8992d279618d03501f8cc963d92a Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Fri, 7 Apr 2023 19:03:04 +0530 Subject: [PATCH 044/144] debug --- .../explorer/querychangelog_list.html | 77 ++++++++++--------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/explorer/templates/explorer/querychangelog_list.html b/explorer/templates/explorer/querychangelog_list.html index 5734cc5f..4a84d385 100644 --- a/explorer/templates/explorer/querychangelog_list.html +++ b/explorer/templates/explorer/querychangelog_list.html @@ -1,54 +1,55 @@ {% extends "explorer/base.html" %} {% block sql_explorer_navlinks %} - {% if can_change %} -
  • New Query
    • -
  • Playground
    • -
  • Logs
    • -
  • Change Logs
    • - {% endif %} +{% if can_change %} +
  • New Query
    • +
  • Playground
    • +
  • Logs
    • +
  • Change Logs
    • +{% endif %} {% endblock %} {% block sql_explorer_content %} -

    Recent Changed Query Logs - Page {{ page_obj.number }}

    - +

    Recent Changed Query Logs - Page {{ page_obj.number }}

    +
    - - - - - - - - {% for object in recent_change_logs %} - - - - - - - - - {% endfor %} - -
    Run At Run ByOld SQLNew SQLQuery IDPlayground
    {{ object.run_at|date:"SHORT_DATETIME_FORMAT" }}{{ object.run_by_user.email }}{{ object.old_sql }}{{ object.new_sql }} {% if object.query_id %}Query {{ object.query_id }}{% elif object.is_playground %}Playground{% else %}--{% endif %}Open
    - {% if is_paginated %} - +{% endif %} +{% endblock %} \ No newline at end of file From 0ec574d54f97d2f1e79615b79c3bf3b048745e96 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Fri, 7 Apr 2023 20:58:04 +0530 Subject: [PATCH 045/144] debug --- explorer/app_settings.py | 3 ++- explorer/utils.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/explorer/app_settings.py b/explorer/app_settings.py index 2104f8b6..1ec47a91 100644 --- a/explorer/app_settings.py +++ b/explorer/app_settings.py @@ -2,7 +2,8 @@ # Required EXPLORER_CONNECTION_NAME = getattr(settings, 'EXPLORER_CONNECTION_NAME', None) -EXPLORER_CONNECTION_PII_NAME = getattr(settings, 'EXPLORER_CONNECTION_PII_NAME', None) +EXPLORER_CONNECTION_NAME_PII = getattr( + settings, 'EXPLORER_CONNECTION_PII_NAME', None) # Change the behavior of explorer EXPLORER_SQL_BLACKLIST = getattr(settings, 'EXPLORER_SQL_BLACKLIST', ('ALTER', 'RENAME ', 'DROP', 'TRUNCATE', 'INSERT INTO', 'UPDATE', 'REPLACE', 'DELETE', 'CREATE TABLE', 'SCHEMA', 'GRANT', 'OWNER TO')) diff --git a/explorer/utils.py b/explorer/utils.py index d518acd3..0cbd7220 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -40,7 +40,7 @@ def get_connection(): def get_connection_pii(): - return connections[app_settings.EXPLORER_CONNECTION_NAME_PII] if app_settings.EXPLORER_CONNECTION_NAME_PII else connection + return connections[app_settings.EXPLORER_CONNECTION_PII_NAME] if app_settings.EXPLORER_CONNECTION_PII_NAME else connection def schema_info(): From f64fc787609270ca7c4a5599c3d667f2199a1b0c Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Fri, 7 Apr 2023 21:29:14 +0530 Subject: [PATCH 046/144] debug again --- explorer/app_settings.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/explorer/app_settings.py b/explorer/app_settings.py index 1ec47a91..4f146c2e 100644 --- a/explorer/app_settings.py +++ b/explorer/app_settings.py @@ -2,30 +2,44 @@ # Required EXPLORER_CONNECTION_NAME = getattr(settings, 'EXPLORER_CONNECTION_NAME', None) -EXPLORER_CONNECTION_NAME_PII = getattr( +EXPLORER_CONNECTION_PII_NAME = getattr( settings, 'EXPLORER_CONNECTION_PII_NAME', None) # Change the behavior of explorer -EXPLORER_SQL_BLACKLIST = getattr(settings, 'EXPLORER_SQL_BLACKLIST', ('ALTER', 'RENAME ', 'DROP', 'TRUNCATE', 'INSERT INTO', 'UPDATE', 'REPLACE', 'DELETE', 'CREATE TABLE', 'SCHEMA', 'GRANT', 'OWNER TO')) -EXPLORER_SQL_WHITELIST = getattr(settings, 'EXPLORER_SQL_WHITELIST', ('CREATED', 'DELETED', 'REGEXP_REPLACE')) +EXPLORER_SQL_BLACKLIST = getattr(settings, 'EXPLORER_SQL_BLACKLIST', ('ALTER', 'RENAME ', 'DROP', 'TRUNCATE', + 'INSERT INTO', 'UPDATE', 'REPLACE', 'DELETE', 'CREATE TABLE', 'SCHEMA', 'GRANT', 'OWNER TO')) +EXPLORER_SQL_WHITELIST = getattr( + settings, 'EXPLORER_SQL_WHITELIST', ('CREATED', 'DELETED', 'REGEXP_REPLACE')) EXPLORER_DEFAULT_ROWS = getattr(settings, 'EXPLORER_DEFAULT_ROWS', 1000) -EXPLORER_SCHEMA_EXCLUDE_APPS = getattr(settings, 'EXPLORER_SCHEMA_EXCLUDE_APPS', ('django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.admin')) +EXPLORER_SCHEMA_EXCLUDE_APPS = getattr(settings, 'EXPLORER_SCHEMA_EXCLUDE_APPS', ( + 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.admin')) EXPLORER_TRANSFORMS = getattr(settings, 'EXPLORER_TRANSFORMS', []) -EXPLORER_PERMISSION_VIEW = getattr(settings, 'EXPLORER_PERMISSION_VIEW', lambda u: u.is_staff) -EXPLORER_PERMISSION_CHANGE = getattr(settings, 'EXPLORER_PERMISSION_CHANGE', lambda u: u.is_staff) -EXPLORER_RECENT_QUERY_COUNT = getattr(settings, 'EXPLORER_RECENT_QUERY_COUNT', 10) +EXPLORER_PERMISSION_VIEW = getattr( + settings, 'EXPLORER_PERMISSION_VIEW', lambda u: u.is_staff) +EXPLORER_PERMISSION_CHANGE = getattr( + settings, 'EXPLORER_PERMISSION_CHANGE', lambda u: u.is_staff) +EXPLORER_RECENT_QUERY_COUNT = getattr( + settings, 'EXPLORER_RECENT_QUERY_COUNT', 10) CSV_DELIMETER = getattr(settings, "EXPLORER_CSV_DELIMETER", ",") # API access EXPLORER_TOKEN = getattr(settings, 'EXPLORER_TOKEN', 'CHANGEME') # These are callable to aid testability by dodging the settings cache. # There is surely a better pattern for this, but this'll hold for now. -EXPLORER_GET_USER_QUERY_VIEWS = lambda: getattr(settings, 'EXPLORER_USER_QUERY_VIEWS', {}) -EXPLORER_TOKEN_AUTH_ENABLED = lambda: getattr(settings, 'EXPLORER_TOKEN_AUTH_ENABLED', False) + + +def EXPLORER_GET_USER_QUERY_VIEWS(): return getattr( + settings, 'EXPLORER_USER_QUERY_VIEWS', {}) + + +def EXPLORER_TOKEN_AUTH_ENABLED(): return getattr( + settings, 'EXPLORER_TOKEN_AUTH_ENABLED', False) + # Async task related. Note that the EMAIL_HOST settings must be set up for email to work. ENABLE_TASKS = getattr(settings, "EXPLORER_TASKS_ENABLED", False) S3_ACCESS_KEY = getattr(settings, "EXPLORER_S3_ACCESS_KEY", None) S3_SECRET_KEY = getattr(settings, "EXPLORER_S3_SECRET_KEY", None) S3_BUCKET = getattr(settings, "EXPLORER_S3_BUCKET", None) -FROM_EMAIL = getattr(settings, 'EXPLORER_FROM_EMAIL', 'django-sql-explorer@example.com') \ No newline at end of file +FROM_EMAIL = getattr(settings, 'EXPLORER_FROM_EMAIL', + 'django-sql-explorer@example.com') From 34f84baa2b61217f83ae18257739010a1272e2c4 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Mon, 10 Apr 2023 11:15:18 +0530 Subject: [PATCH 047/144] debug --- explorer/models.py | 4 +++- explorer/templates/explorer/play.html | 4 ++-- explorer/templates/explorer/query_list.html | 4 ++-- explorer/templates/explorer/querychangelog_list.html | 2 +- explorer/templates/explorer/querylog_list.html | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/explorer/models.py b/explorer/models.py index 1a55cd3d..cee8e023 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -219,8 +219,10 @@ def process_rows(self): r[ix] = t.format(str(r[ix])) def execute_query(self): + # can change connectiion type here to use different role --> get_connection_pii() if (self.connection_type): - # can change connectiion type here to use different role --> get_connection_pii() + logger.info( + "----------------------------pii-connection-------------------------") conn = get_connection_pii() else: conn = get_connection() diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index ba582845..32e4a259 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -2,8 +2,8 @@ {% block sql_explorer_navlinks %} {% if can_change %} - -
  • Play
    • +
  • New Query
    • +
  • Playground
  • Logs
  • Change Logs
    • {% endif %} diff --git a/explorer/templates/explorer/query_list.html b/explorer/templates/explorer/query_list.html index bc747d8d..7efd0695 100644 --- a/explorer/templates/explorer/query_list.html +++ b/explorer/templates/explorer/query_list.html @@ -4,7 +4,7 @@ {% block sql_explorer_navlinks %} {% if can_change %}
  • New Query
    • -
  • Play
    • +
  • ground
  • Logs
  • Change Logs
    • {% endif %} @@ -54,7 +54,7 @@

    All Queries

    {% endif %} CSV {% if can_change %} - Play + Playground Delete {% endif %} Run Count diff --git a/explorer/templates/explorer/querychangelog_list.html b/explorer/templates/explorer/querychangelog_list.html index 4a84d385..cdd33adf 100644 --- a/explorer/templates/explorer/querychangelog_list.html +++ b/explorer/templates/explorer/querychangelog_list.html @@ -19,7 +19,7 @@

    Recent Changed Query Logs - Page {{ page_obj.number }}

    Old SQL New SQL Query ID - Play + Playground diff --git a/explorer/templates/explorer/querylog_list.html b/explorer/templates/explorer/querylog_list.html index 32418f2d..c0ca59ab 100644 --- a/explorer/templates/explorer/querylog_list.html +++ b/explorer/templates/explorer/querylog_list.html @@ -3,7 +3,7 @@ {% block sql_explorer_navlinks %} {% if can_change %}
  • New Query
    • -
  • Play
    • +
  • Playground
  • Logs
  • Change Logs
    • {% endif %} From 715270f7efc65decf16fe45ea5e91177f234aab5 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Mon, 10 Apr 2023 13:01:15 +0530 Subject: [PATCH 048/144] debug --- explorer/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/explorer/models.py b/explorer/models.py index cee8e023..0a1ef8f2 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -25,8 +25,6 @@ class Query(models.Model): last_run_date = models.DateTimeField(auto_now=True) snapshot = models.BooleanField( default=False, help_text="Include in snapshot task (if enabled)") - connection = models.BooleanField( - default=False, help_text="use to select connection type") def __init__(self, *args, **kwargs): self.params = kwargs.get('params') @@ -225,6 +223,8 @@ def execute_query(self): "----------------------------pii-connection-------------------------") conn = get_connection_pii() else: + logger.info( + "----------------------------non-pii-connection-------------------------") conn = get_connection() cursor = conn.cursor() start_time = time() From 16f879d0d80fb6f8c7eb9d8178e2f0fc02292639 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Mon, 10 Apr 2023 14:09:48 +0530 Subject: [PATCH 049/144] test --- explorer/templates/explorer/play.html | 6 +++--- explorer/templates/explorer/query_list.html | 8 ++++---- explorer/templates/explorer/querychangelog_list.html | 6 +++--- explorer/templates/explorer/querylog_list.html | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 32e4a259..fd25d1d6 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -2,10 +2,10 @@ {% block sql_explorer_navlinks %} {% if can_change %} -
  • New Query
    • +
  • New Query
  • Playground
    • -
  • Logs
    • -
  • Change Logs
    • +
  • Logs
    • +
  • Change Logs
    • {% endif %} {% endblock %} diff --git a/explorer/templates/explorer/query_list.html b/explorer/templates/explorer/query_list.html index 7efd0695..0e6bc9bb 100644 --- a/explorer/templates/explorer/query_list.html +++ b/explorer/templates/explorer/query_list.html @@ -3,10 +3,10 @@ {% block sql_explorer_navlinks %} {% if can_change %} -
  • New Query
    • -
  • ground
    • -
  • Logs
    • -
  • Change Logs
    • +
  • New Query
    • +
  • ground
    • +
  • Logs
    • +
  • Change Logs
    • {% endif %} {% endblock %} diff --git a/explorer/templates/explorer/querychangelog_list.html b/explorer/templates/explorer/querychangelog_list.html index cdd33adf..63444395 100644 --- a/explorer/templates/explorer/querychangelog_list.html +++ b/explorer/templates/explorer/querychangelog_list.html @@ -2,9 +2,9 @@ {% block sql_explorer_navlinks %} {% if can_change %} -
  • New Query
    • -
  • Playground
    • -
  • Logs
    • +
  • New Query
    • +
  • Playground
    • +
  • Logs
  • Change Logs
    • {% endif %} {% endblock %} diff --git a/explorer/templates/explorer/querylog_list.html b/explorer/templates/explorer/querylog_list.html index c0ca59ab..7a657288 100644 --- a/explorer/templates/explorer/querylog_list.html +++ b/explorer/templates/explorer/querylog_list.html @@ -2,10 +2,10 @@ {% block sql_explorer_navlinks %} {% if can_change %} -
  • New Query
    • -
  • Playground
    • +
  • New Query
    • +
  • Playground
  • Logs
    • -
  • Change Logs
    • +
  • Change Logs
    • {% endif %} {% endblock %} From 8aff03221d60e39f9cf40e6a2e48ac7d39ad8c6a Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Mon, 10 Apr 2023 14:12:27 +0530 Subject: [PATCH 050/144] test --- explorer/templates/explorer/play.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index fd25d1d6..563d6696 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -1,12 +1,12 @@ {% extends "explorer/base.html" %} {% block sql_explorer_navlinks %} -{% if can_change %} -
  • New Query
    • -
  • Playground
    • -
  • Logs
    • -
  • Change Logs
    • -{% endif %} + {% if can_change %} +
  • New Query
    • +
  • Playground
    • +
  • Logs
    • +
  • Change Logs
    • + {% endif %} {% endblock %} {% block sql_explorer_content %} From 6f2645efcba3c996f3cacbae7eb70e632ca53860 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Mon, 10 Apr 2023 17:23:16 +0530 Subject: [PATCH 051/144] testing --- explorer/models.py | 11 ++++++----- explorer/templates/explorer/querychangelog_list.html | 6 +++--- explorer/utils.py | 2 ++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/explorer/models.py b/explorer/models.py index 0a1ef8f2..5eba9db1 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -67,6 +67,7 @@ def execute(self): return ret def execute_pii(self): + print("execute_pii") ret = self.execute_query_only(True) ret.process() return ret @@ -147,7 +148,7 @@ class QueryResult(object): def __init__(self, sql, connection_type=None): self.sql = sql - logger.info( + print( "----------------------------debug1-------------------------") if (connection_type): self.connection_type = connection_type @@ -160,9 +161,9 @@ def __init__(self, sql, connection_type=None): self._data = [list(r) for r in cursor.fetchall()] self.duration = duration - logger.info( + print( "----------------------------debug-------------------------") - logger.info(self) + cursor.close() @@ -219,11 +220,11 @@ def process_rows(self): def execute_query(self): # can change connectiion type here to use different role --> get_connection_pii() if (self.connection_type): - logger.info( + print( "----------------------------pii-connection-------------------------") conn = get_connection_pii() else: - logger.info( + print( "----------------------------non-pii-connection-------------------------") conn = get_connection() cursor = conn.cursor() diff --git a/explorer/templates/explorer/querychangelog_list.html b/explorer/templates/explorer/querychangelog_list.html index 63444395..cdd33adf 100644 --- a/explorer/templates/explorer/querychangelog_list.html +++ b/explorer/templates/explorer/querychangelog_list.html @@ -2,9 +2,9 @@ {% block sql_explorer_navlinks %} {% if can_change %} -
  • New Query
    • -
  • Playground
    • -
  • Logs
    • +
  • New Query
    • +
  • Playground
    • +
  • Logs
  • Change Logs
    • {% endif %} {% endblock %} diff --git a/explorer/utils.py b/explorer/utils.py index 0cbd7220..78f7ba5b 100644 --- a/explorer/utils.py +++ b/explorer/utils.py @@ -40,6 +40,8 @@ def get_connection(): def get_connection_pii(): + print("-------------------explorer_pii_connection---------------------") + print(app_settings.EXPLORER_CONNECTION_PII_NAME) return connections[app_settings.EXPLORER_CONNECTION_PII_NAME] if app_settings.EXPLORER_CONNECTION_PII_NAME else connection From fa02c3a6f688058e500ffb82a22d34dc54f0fb81 Mon Sep 17 00:00:00 2001 From: CodeNinja04 Date: Tue, 11 Apr 2023 12:59:21 +0530 Subject: [PATCH 052/144] code refactored --- explorer/models.py | 13 +++---------- explorer/templates/explorer/play.html | 4 ++-- explorer/templates/explorer/query_list.html | 14 +++++++------- .../templates/explorer/querychangelog_list.html | 12 ++++++------ explorer/templates/explorer/querylog_list.html | 4 ++-- explorer/utils.py | 10 +++++++--- 6 files changed, 27 insertions(+), 30 deletions(-) diff --git a/explorer/models.py b/explorer/models.py index 5eba9db1..c8e1c1c0 100644 --- a/explorer/models.py +++ b/explorer/models.py @@ -148,8 +148,6 @@ class QueryResult(object): def __init__(self, sql, connection_type=None): self.sql = sql - print( - "----------------------------debug1-------------------------") if (connection_type): self.connection_type = connection_type else: @@ -161,10 +159,6 @@ def __init__(self, sql, connection_type=None): self._data = [list(r) for r in cursor.fetchall()] self.duration = duration - print( - "----------------------------debug-------------------------") - - cursor.close() self._headers = self._get_headers() @@ -220,12 +214,11 @@ def process_rows(self): def execute_query(self): # can change connectiion type here to use different role --> get_connection_pii() if (self.connection_type): - print( - "----------------------------pii-connection-------------------------") + logger.info( + "pii-connection") conn = get_connection_pii() else: - print( - "----------------------------non-pii-connection-------------------------") + logger.info("non-pii-connection") conn = get_connection() cursor = conn.cursor() start_time = time() diff --git a/explorer/templates/explorer/play.html b/explorer/templates/explorer/play.html index 563d6696..396d4d47 100644 --- a/explorer/templates/explorer/play.html +++ b/explorer/templates/explorer/play.html @@ -27,7 +27,7 @@

    Playground Test

    whatsapp2_whatsapp2inboundmessage, whatsapp2_whatsapp2outboundmessage and voice_compliance_businessoutreach. Please avoid using `select *` for these tables.

    - {% + {% csrf_token %} {% if error %}
    {{ error|escape }}
    @@ -40,7 +40,7 @@

    Playground Test

    {% if ql_id %}
    -
    {% endif %} diff --git a/explorer/templates/explorer/query_list.html b/explorer/templates/explorer/query_list.html index 0e6bc9bb..d81d3e4b 100644 --- a/explorer/templates/explorer/query_list.html +++ b/explorer/templates/explorer/query_list.html @@ -24,10 +24,10 @@

    {{ recent_queries|length }} Most Recently Used

    {% for object in recent_queries %} - {{ object.title }} + {{ object.title }} {{ object.last_run_date|date:"SHORT_DATETIME_FORMAT" }} - + {% endfor %} @@ -72,7 +72,7 @@

    All Queries

    {% else %} - {% if object.is_in_category %} → {% endif %}{{ object.title }} {{ object.created_at|date:"SHORT_DATE_FORMAT" }} @@ -82,20 +82,20 @@

    All Queries

    {% if tasks_enabled %} -