@@ -37,11 +60,12 @@
Playground
{% endif %}
+ {% if lag_exists %}
+
Warning! The query results might be outdated since some data was last updated around {{ replication_lag }}.
+ {% endif %}
+{% endif %}
+{% endblock %}
\ No newline at end of file
diff --git a/explorer/tests/settings.py b/explorer/tests/settings.py
index f3d0e3ae..b7296f07 100644
--- a/explorer/tests/settings.py
+++ b/explorer/tests/settings.py
@@ -10,7 +10,7 @@
'PASSWORD': '',
'HOST': '',
'PORT': '',
- }
+ },
}
ROOT_URLCONF = 'explorer.tests.urls'
@@ -48,7 +48,7 @@
STATIC_URL = '/static/'
-MIDDLEWARE_CLASSES = ('django.contrib.sessions.middleware.SessionMiddleware',
+MIDDLEWARE = ('django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware')
EXPLORER_USER_QUERY_VIEWS = {}
diff --git a/explorer/tests/test_utils.py b/explorer/tests/test_utils.py
index c99dcc6d..ea628cc5 100644
--- a/explorer/tests/test_utils.py
+++ b/explorer/tests/test_utils.py
@@ -128,4 +128,4 @@ def test_writing_unicode(self):
headers = ['a', None]
data = [[1, None], [u"Jenét", '1']]
res = write_csv(headers, data).getvalue()
- self.assertEqual(res, 'a,\r\n1,\r\nJenét,1\r\n')
\ No newline at end of file
+ self.assertEqual(res, 'a,\r\n1,\r\nJenét,1\r\n')
diff --git a/explorer/tests/test_views.py b/explorer/tests/test_views.py
index 6fc6aafd..6c05d081 100644
--- a/explorer/tests/test_views.py
+++ b/explorer/tests/test_views.py
@@ -1,10 +1,10 @@
from django.test import TestCase
-from django.core.urlresolvers import reverse
+from django.urls import reverse
from django.contrib.auth.models import User
from django.conf import settings
from django.forms.models import model_to_dict
from explorer.tests.factories import SimpleQueryFactory, QueryLogFactory
-from explorer.models import Query, QueryLog, MSG_FAILED_BLACKLIST
+from explorer.models import Query, QueryLog, QueryChangeLog, MSG_FAILED_BLACKLIST
from explorer.views import user_can_see_query
from explorer.app_settings import EXPLORER_TOKEN
from mock import Mock, patch
@@ -426,4 +426,33 @@ def test_is_playground(self):
self.assertTrue(QueryLog(sql='foo').is_playground)
q = SimpleQueryFactory()
- self.assertFalse(QueryLog(sql='foo', query_id=q.id).is_playground)
\ No newline at end of file
+ self.assertFalse(QueryLog(sql='foo', query_id=q.id).is_playground)
+
+
+class TestQueryChangeLog(TestCase):
+
+ def setUp(self):
+ self.user = User.objects.create_superuser('admin', 'admin@admin.com', 'pwd')
+ self.client.login(username='admin', password='pwd')
+
+ def test_admin_required(self):
+ self.client.logout()
+ resp = self.client.get(reverse("explorer_change_logs"))
+ self.assertTemplateUsed(resp, 'admin/login.html')
+
+ def test_query_change_saves_to_change_log(self):
+ query = SimpleQueryFactory()
+ data = model_to_dict(query)
+ data['sql'] = 'select 12345;'
+ self.client.post(reverse("query_detail", kwargs={'query_id': query.id}), data)
+ data['sql'] = 'select 67890;'
+ self.client.post(reverse("query_detail", kwargs={'query_id': query.id}), data)
+ resp = self.client.get(reverse("explorer_change_logs"))
+ self.assertContains(resp, 'select 12345;')
+ self.assertContains(resp, 'select 67890;')
+
+ def test_is_playground(self):
+ self.assertTrue(QueryChangeLog(old_sql='foo', new_sql='bar').is_playground)
+
+ q = SimpleQueryFactory()
+ self.assertFalse(QueryChangeLog(old_sql='foo', new_sql='bar', query_id=q.id).is_playground)
diff --git a/explorer/tests/urls.py b/explorer/tests/urls.py
index 5d729a95..5021e057 100644
--- a/explorer/tests/urls.py
+++ b/explorer/tests/urls.py
@@ -1,9 +1,7 @@
-from django.conf.urls import patterns, url, include
+from django.conf.urls import url, include
from django.contrib import admin
from explorer.urls import urlpatterns
admin.autodiscover()
-urlpatterns += patterns('',
- url(r'^admin/', include(admin.site.urls)),
-)
\ No newline at end of file
+urlpatterns += [url(r"^admin/", include(admin.site.urls))]
diff --git a/explorer/urls.py b/explorer/urls.py
index d270d96c..e88dadbe 100644
--- a/explorer/urls.py
+++ b/explorer/urls.py
@@ -6,6 +6,7 @@
DeleteQueryView,
ListQueryView,
ListQueryLogView,
+ ListQueryChangeLogView,
download_query,
view_csv_query,
email_csv_query,
@@ -24,6 +25,7 @@
url(r'play/$', PlayQueryView.as_view(), name='explorer_playground'),
url(r'csv$', download_csv_from_sql, name='generate_csv'),
url(r'schema/$', schema, name='explorer_schema'),
+ url(r'changelogs/$', ListQueryChangeLogView.as_view(), name='explorer_change_logs'),
url(r'logs/$', ListQueryLogView.as_view(), name='explorer_logs'),
url(r'format/$', format_sql, name='format_sql'),
url(r'^$', ListQueryView.as_view(), name='explorer_index'),
diff --git a/explorer/utils.py b/explorer/utils.py
index 5cf2541a..8f3c857e 100644
--- a/explorer/utils.py
+++ b/explorer/utils.py
@@ -1,36 +1,75 @@
-
-
+from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth.views import login
+from django.contrib.admin.forms import AdminAuthenticationForm
+import datetime
+import sqlparse
+from ago import human
+from six.moves import cStringIO
+from django.http import HttpResponse
+from django.db import connections, connection, DatabaseError
+from explorer import app_settings
+import string
+import re
import functools
import sys
+import logging
+
+from explorer.constants import PII_MASKING_PATTERN_REPLACEMENT_DICT, ALLOW_PHONE_NUMBER_MASKING_GROUP_ID, \
+ PATTERN_FOR_FINDING_EMAIL, PATTERN_FOR_FINDING_PHONE_NUMBER
+
+logger = logging.getLogger(__name__)
+
PY3 = sys.version_info[0] == 3
if PY3:
import csv
else:
import unicodecsv as csv
-import re
-import string
-from explorer import app_settings
-from django.db import connections, connection, DatabaseError
-from django.http import HttpResponse
-from six.moves import cStringIO
-import sqlparse
EXPLORER_PARAM_TOKEN = "$$"
+REPLICATION_LAG_THRESHOLD_VALUE_IN_MINUTES = 3
+
+
# SQL Specific Things
def passes_blacklist(sql):
- clean = functools.reduce(lambda sql, term: sql.upper().replace(term, ""), [t.upper() for t in app_settings.EXPLORER_SQL_WHITELIST], sql)
- fails = [bl_word for bl_word in app_settings.EXPLORER_SQL_BLACKLIST if bl_word in clean.upper()]
+ clean = functools.reduce(lambda sql, term: sql.upper().replace(term, ""), [
+ t.upper() for t in app_settings.EXPLORER_SQL_WHITELIST], sql)
+ fails = [
+ bl_word for bl_word in app_settings.EXPLORER_SQL_BLACKLIST if bl_word in clean.upper()]
return not any(fails), fails
def get_connection():
+ logger.info("explorer_connection succesfull")
return connections[app_settings.EXPLORER_CONNECTION_NAME] if app_settings.EXPLORER_CONNECTION_NAME else connection
+def get_explorer_master_db_connection():
+ logger.info("get_explorer_master_db_connection successful")
+ return connections[
+ app_settings.EXPLORER_MASTER_DB_CONNECTION_NAME] if app_settings.EXPLORER_MASTER_DB_CONNECTION_NAME else connection
+
+
+def get_connection_pii():
+ logger.info("explorer_pii_connection succesfull")
+ return connections[app_settings.EXPLORER_CONNECTION_PII_NAME] if app_settings.EXPLORER_CONNECTION_PII_NAME else connection
+
+
+def get_master_db_connection():
+ logger.info("explorer_pii_connection succesfull")
+ return connections[
+ app_settings.EXPLORER_CONNECTION_PII_NAME] if app_settings.EXPLORER_CONNECTION_PII_NAME else connection
+
+
+def get_connection_asyncapi_db():
+ logger.info("Connecting with async-api DB")
+ return connections[
+ app_settings.EXPLORER_CONNECTION_ASYNC_API_DB_NAME] if app_settings.EXPLORER_CONNECTION_ASYNC_API_DB_NAME else connection
+
+
def schema_info():
"""
Construct schema information via introspection of the django models in the database.
@@ -48,6 +87,9 @@ def schema_info():
"""
from django.apps import apps
+ import logging
+
+ logger = logging.getLogger(__name__)
ret = []
@@ -56,18 +98,29 @@ def schema_info():
for model_name, model in apps.get_app_config(label).models.items():
friendly_model = "%s -> %s" % (app.name, model._meta.object_name)
ret.append((
- friendly_model,
- model._meta.db_table,
- [_format_field(f) for f in model._meta.fields]
- ))
-
- # Do the same thing for many_to_many fields. These don't show up in the field list of the model
- # because they are stored as separate "through" relations and have their own tables
- ret += [(
- friendly_model,
- m2m.rel.through._meta.db_table,
- [_format_field(f) for f in m2m.rel.through._meta.fields]
- ) for m2m in model._meta.many_to_many]
+ friendly_model,
+ model._meta.db_table,
+ [_format_field(f) for f in model._meta.fields]
+ ))
+
+ try:
+ # Loop over Many-to-Many relationships
+ for m2m in model._meta.many_to_many:
+ through_model = m2m.remote_field.through
+ ret.append(
+ (
+ friendly_model,
+ through_model._meta.db_table,
+ [
+ _format_field(f)
+ for f in through_model._meta.fields
+ ],
+ )
+ )
+ except Exception as e:
+ logger.error(
+ "Error while processing Many-to-Many relationships: %s" % e
+ )
return sorted(ret, key=lambda t: t[1])
@@ -119,14 +172,14 @@ def get_filename_for_title(title):
return filename
-def build_stream_response(query, delim=None):
- data = csv_report(query, delim).getvalue()
+def build_stream_response(query, delim=None, user=None):
+ data = csv_report(query, delim, user).getvalue()
response = HttpResponse(data, content_type='text')
return response
-def build_download_response(query, delim=None):
- data = csv_report(query, delim).getvalue()
+def build_download_response(query, delim=None, user=None):
+ data = csv_report(query, delim, user).getvalue()
response = HttpResponse(data, content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="%s.csv"' % (
get_filename_for_title(query.title)
@@ -135,18 +188,15 @@ def build_download_response(query, delim=None):
return response
-def csv_report(query, delim=None):
+def csv_report(query, delim=None, user=None):
try:
- res = query.execute_query_only()
+ res = query.execute_query_only(executing_user=user)
return write_csv(res.headers, res.data, delim)
except DatabaseError as e:
return str(e)
# Helpers
-from django.contrib.admin.forms import AdminAuthenticationForm
-from django.contrib.auth.views import login
-from django.contrib.auth import REDIRECT_FIELD_NAME
def safe_admin_login_prompt(request):
@@ -236,4 +286,132 @@ def get_s3_connection():
import tinys3
return tinys3.Connection(app_settings.S3_ACCESS_KEY,
app_settings.S3_SECRET_KEY,
- default_bucket=app_settings.S3_BUCKET)
\ No newline at end of file
+ default_bucket=app_settings.S3_BUCKET)
+
+
+def compare_sql(old_sql, new_sql):
+ """
+ Compares whether two sql queries are the
+ same after formatting them
+ """
+ return fmt_sql(old_sql) == fmt_sql(new_sql)
+
+
+def check_replication_lag():
+ """
+ Check if a replication lag exists
+ :returns: True and the replication lag interval if it
+ exceeds 3 minutes, else returns False and None
+ """
+ conn = get_connection()
+ cursor = conn.cursor()
+
+ cursor.execute(
+ "SELECT now() - pg_last_xact_replay_timestamp() AS replication_lag")
+ replication_lag = cursor.fetchone()[0]
+
+ threshold_value = datetime.timedelta(
+ minutes=REPLICATION_LAG_THRESHOLD_VALUE_IN_MINUTES)
+
+ if not replication_lag or replication_lag <= threshold_value:
+ return False, None
+
+ return True, human(replication_lag, 4)
+
+
+def should_route_to_asyncapi_db(sql):
+ request_log_tables = [
+ "request_log_requestlog",
+ "request_log_requestlogdata",
+ "temp_request_log_requestlog_customer",
+ "temp_request_log_requestlogdata_customer",
+ ]
+ pattern = r"\b(?:%s)\b" % "|".join(map(re.escape, request_log_tables))
+ match = re.search(pattern, sql)
+ if match:
+ return True
+
+ return False
+
+
+def mask_string(string_to_masked):
+ """
+ Replace a string with a dictionary of regex patterns and replacements
+ Param 1: string that needs to be masked
+
+ Eg.
+ Following is the PII_MASKING_PATTERN_REPLACEMENT_DICT as of now
+ {
+ r"(?:\+?\d{1,3}|0)?([6-9]\d{9})\b": "XXXXXXXXXXX", # For masking phone number
+ r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b": "XXX@XXX.com", # For masking email
+ }
+ It would mask the string:
+ 'My number is +919191919191, their number is 9191919191, my email is abc@abc.com, their email is xyz@pq.in.'
+ to:
+ 'My number is XXXXXXXXXXX, their number is XXXXXXXXXXX, my email is XXX@XXX.com, their email is XXX@XXX.com.'
+ and return it.
+ """
+
+ for pattern, replacement in PII_MASKING_PATTERN_REPLACEMENT_DICT.items():
+ string_to_masked = re.sub(pattern, replacement, string_to_masked)
+ return string_to_masked
+
+
+def get_masked_value(value: str):
+ """
+ This function will return the masked value depending upon the length of the value.
+ For eg.
+ 1. value = None => masked_value = None
+ 1. value = "son" => masked_value = "XXX"
+ 2. value = "sunny" => masked_value = "sXXXy"
+ 3. value = "my name is kunal" => masked_value = "myXXXXXXXXXXXal"
+ """
+
+ if not value:
+ return None
+
+ if len(value) <= 3:
+ return "".join("X" for _ in range(len(value)))
+ elif 4 <= len(value) <= 5:
+ return value[0] + "".join("X" for _ in range(len(value) - 2)) + value[-1]
+ else:
+ return value[:2] + "".join("X" for _ in range(len(value) - 2)) + value[-2:]
+
+
+def get_masked_email(email: str) -> str:
+ if not email or "@" not in email:
+ return email
+
+ local_part, domain_part = email.split("@")
+ masked_local_part = get_masked_value(local_part)
+ return masked_local_part + "@" + "".join("X" for _ in range(len(domain_part)))
+
+
+def get_masked_phone_number(phone_number: str) -> str:
+ if not phone_number or len(phone_number) < 10:
+ return phone_number
+
+ return "".join("X" for _ in range(len(phone_number) - 4)) + phone_number[-4:]
+
+
+def mask_player_pii(data: str) -> str:
+ if not data:
+ return data
+
+ matched_phone_numbers = re.findall(PATTERN_FOR_FINDING_PHONE_NUMBER, data)
+ for phone_number in matched_phone_numbers:
+ data = data.replace(phone_number, get_masked_phone_number(phone_number))
+
+ matched_emails = re.findall(PATTERN_FOR_FINDING_EMAIL, data)
+ for email in matched_emails:
+ data = data.replace(email, get_masked_email(email))
+
+ return data
+
+
+def is_pii_masked_for_user(user):
+ """
+ Check if the user has permission to view masked phone numbers
+ """
+ user_group_ids = user.groups.all().values_list('id', flat=True)
+ return ALLOW_PHONE_NUMBER_MASKING_GROUP_ID not in user_group_ids
diff --git a/explorer/views.py b/explorer/views.py
index 419525de..cba6755b 100644
--- a/explorer/views.py
+++ b/explorer/views.py
@@ -1,5 +1,8 @@
+from django.views.decorators.csrf import csrf_exempt
+
from explorer.tasks import execute_query
import six
+import logging
from django.http.response import HttpResponseRedirect
from django.shortcuts import render_to_response, get_object_or_404
@@ -9,49 +12,54 @@
from django.views.generic.edit import CreateView, DeleteView
from django.views.decorators.http import require_POST, require_GET
from django.utils.decorators import method_decorator
-from django.core.urlresolvers import reverse_lazy
+from django.urls import reverse_lazy
from django.forms.models import model_to_dict
from django.http import HttpResponse
from django.db import DatabaseError
from django.db.models import Count
from django.forms import ValidationError
+from django.utils.translation import gettext_lazy as _
-from explorer.models import Query, QueryLog, MSG_FAILED_BLACKLIST
+from explorer.models import Query, QueryLog, QueryChangeLog, MSG_FAILED_BLACKLIST
from explorer import app_settings
from explorer.forms import QueryForm
-from explorer.utils import url_get_rows,\
- url_get_query_id,\
- url_get_log_id,\
- schema_info,\
- url_get_params,\
- safe_admin_login_prompt,\
- build_download_response,\
- build_stream_response,\
- user_can_see_query,\
- fmt_sql,\
- allowed_query_pks,\
- url_get_show
+from explorer.utils import url_get_rows, \
+ url_get_query_id, \
+ url_get_log_id, \
+ schema_info, \
+ url_get_params, \
+ safe_admin_login_prompt, \
+ build_download_response, \
+ build_stream_response, \
+ user_can_see_query, \
+ fmt_sql, \
+ allowed_query_pks, \
+ url_get_show, \
+ compare_sql, \
+ check_replication_lag
try:
from collections import Counter
except:
from counter import Counter
-
import re
import json
from functools import wraps
+logger = logging.getLogger(__name__)
+
def view_permission(f):
@wraps(f)
def wrap(request, *args, **kwargs):
- if not app_settings.EXPLORER_PERMISSION_VIEW(request.user)\
- and not user_can_see_query(request, kwargs)\
+ if not app_settings.EXPLORER_PERMISSION_VIEW(request.user) \
+ and not user_can_see_query(request, kwargs) \
and not (app_settings.EXPLORER_TOKEN_AUTH_ENABLED()
and request.META.get('HTTP_X_API_TOKEN') == app_settings.EXPLORER_TOKEN):
return safe_admin_login_prompt(request)
return f(request, *args, **kwargs)
+
return wrap
@@ -61,10 +69,11 @@ def wrap(request, *args, **kwargs):
def view_permission_list(f):
@wraps(f)
def wrap(request, *args, **kwargs):
- if not app_settings.EXPLORER_PERMISSION_VIEW(request.user)\
+ if not app_settings.EXPLORER_PERMISSION_VIEW(request.user) \
and not allowed_query_pks(request.user.id):
return safe_admin_login_prompt(request)
return f(request, *args, **kwargs)
+
return wrap
@@ -74,6 +83,7 @@ def wrap(request, *args, **kwargs):
if not app_settings.EXPLORER_PERMISSION_CHANGE(request.user):
return safe_admin_login_prompt(request)
return f(request, *args, **kwargs)
+
return wrap
@@ -119,14 +129,17 @@ def email_csv_query(request, query_id):
def _csv_response(request, query_id, stream=False, delim=None):
query = get_object_or_404(Query, pk=query_id)
query.params = url_get_params(request)
- return build_stream_response(query, delim) if stream else build_download_response(query, delim)
+ return build_stream_response(query, delim, user=request.user) if stream else build_download_response(query, delim,
+ user=request.user)
+@csrf_exempt
@change_permission
@require_POST
def download_csv_from_sql(request):
sql = request.POST.get('sql')
- return build_download_response(Query(sql=sql, title="Playground", params=url_get_params(request)))
+ return build_download_response(Query(sql=sql, title="Playground", params=url_get_params(request)),
+ user=request.user)
@change_permission
@@ -135,13 +148,17 @@ def schema(request):
return render_to_response('explorer/schema.html', {'schema': schema_info()})
+@csrf_exempt
@require_POST
def format_sql(request):
+ if not (request.user.is_authenticated()):
+ return HttpResponse(status=403)
sql = request.POST.get('sql', '')
formatted = fmt_sql(sql)
return HttpResponse(json.dumps({"formatted": formatted}), content_type="application/json")
+@method_decorator(csrf_exempt, name='dispatch')
class ListQueryView(ExplorerContextMixin, ListView):
@method_decorator(view_permission_list)
@@ -151,7 +168,8 @@ def dispatch(self, *args, **kwargs):
def get_context_data(self, **kwargs):
context = super(ListQueryView, self).get_context_data(**kwargs)
context['object_list'] = self._build_queries_and_headers()
- context['recent_queries'] = self.get_queryset().order_by('-last_run_date')[:app_settings.EXPLORER_RECENT_QUERY_COUNT]
+ context['recent_queries'] = self.get_queryset().order_by(
+ '-last_run_date')[:app_settings.EXPLORER_RECENT_QUERY_COUNT]
context['tasks_enabled'] = app_settings.ENABLE_TASKS
return context
@@ -159,7 +177,8 @@ def get_queryset(self):
if app_settings.EXPLORER_PERMISSION_VIEW(self.request.user):
qs = Query.objects.prefetch_related('created_by_user').all()
else:
- qs = Query.objects.prefetch_related('created_by_user').filter(pk__in=allowed_query_pks(self.request.user.id))
+ qs = Query.objects.prefetch_related('created_by_user').filter(
+ pk__in=allowed_query_pks(self.request.user.id))
return qs.annotate(run_count=Count('querylog'))
def _build_queries_and_headers(self):
@@ -209,6 +228,7 @@ def _build_queries_and_headers(self):
model = Query
+@method_decorator(csrf_exempt, name='dispatch')
class ListQueryLogView(ExplorerContextMixin, ListView):
@method_decorator(view_permission)
@@ -223,6 +243,24 @@ def get_queryset(self):
paginate_by = 20
+@method_decorator(csrf_exempt, name='dispatch')
+class ListQueryChangeLogView(ExplorerContextMixin, ListView):
+
+ @method_decorator(view_permission)
+ def dispatch(self, *args, **kwargs):
+ return super(ListQueryChangeLogView, self).dispatch(*args, **kwargs)
+
+ def get_queryset(self):
+ kwargs = {'old_sql__isnull': False, 'new_sql__isnull': False}
+ return QueryChangeLog.objects.filter(**kwargs).all()
+
+ context_object_name = "recent_change_logs"
+ model = QueryChangeLog
+ paginate_by = 20
+ template_name = 'explorer/querychangelog_list.html'
+
+
+@method_decorator(csrf_exempt, name='dispatch')
class CreateQueryView(ExplorerContextMixin, CreateView):
@method_decorator(change_permission)
@@ -237,6 +275,7 @@ def form_valid(self, form):
template_name = 'explorer/query.html'
+@method_decorator(csrf_exempt, name='dispatch')
class DeleteQueryView(ExplorerContextMixin, DeleteView):
@method_decorator(change_permission)
@@ -247,6 +286,7 @@ def dispatch(self, *args, **kwargs):
success_url = reverse_lazy("explorer_index")
+@method_decorator(csrf_exempt, name='dispatch')
class PlayQueryView(ExplorerContextMixin, View):
@method_decorator(change_permission)
@@ -270,18 +310,23 @@ def post(self, request):
show_results = request.POST.get('show', True)
query = Query(sql=sql, title="Playground")
passes_blacklist, failing_words = query.passes_blacklist()
- error = MSG_FAILED_BLACKLIST % ', '.join(failing_words) if not passes_blacklist else None
+ error = MSG_FAILED_BLACKLIST % ', '.join(
+ failing_words) if not passes_blacklist else None
run_query = not bool(error) if show_results else False
return self.render_with_sql(request, query, run_query=run_query, error=error)
def render(self, request):
- return self.render_template('explorer/play.html', RequestContext(request, {'title': 'Playground'}))
+ return self.render_template(
+ "explorer/play.html", {"title": "Playground", "request": request}
+ )
def render_with_sql(self, request, query, run_query=True, error=None):
- return self.render_template('explorer/play.html', query_viewmodel(request, query, title="Playground"
- , run_query=run_query, error=error))
+ return self.render_template('explorer/play.html',
+ query_viewmodel(request, query, title="Playground", run_query=run_query,
+ error=error))
+@method_decorator(csrf_exempt, name='dispatch')
class QueryView(ExplorerContextMixin, View):
@method_decorator(view_permission)
@@ -291,7 +336,8 @@ def dispatch(self, *args, **kwargs):
def get(self, request, query_id):
query, form = QueryView.get_instance_and_form(request, query_id)
query.save() # updates the modified date
- show = url_get_show(request) # if a query is timing out, it can be useful to nav to /query/id/?show=0
+ # if a query is timing out, it can be useful to nav to /query/id/?show=0
+ show = url_get_show(request)
vm = query_viewmodel(request, query, form=form, run_query=show)
return self.render_template('explorer/query.html', vm)
@@ -300,17 +346,49 @@ def post(self, request, query_id):
return HttpResponseRedirect(
reverse_lazy('query_detail', kwargs={'query_id': query_id})
)
-
+ show = url_get_show(request)
query, form = QueryView.get_instance_and_form(request, query_id)
- success = form.is_valid() and form.save()
- vm = query_viewmodel(request, query, form=form, message="Query saved." if success else None)
+
+ old_sql = query.sql
+ form_isvalid = form.is_valid()
+ if form_isvalid:
+ new_sql = request.POST.get('sql')
+ if not compare_sql(old_sql, new_sql):
+ change_log = QueryChangeLog(
+ old_sql=old_sql,
+ new_sql=new_sql,
+ query=query,
+ run_by_user=request.user,
+ )
+ change_log.save()
+ success = form_isvalid and form.save()
+
+ try:
+ vm = query_viewmodel(
+ request,
+ query,
+ form=form,
+ run_query=show,
+ message=_("Query saved.") if success else "Query not saved"
+ )
+ except ValidationError as ve:
+
+ vm = query_viewmodel(
+ request,
+ query,
+ form=form,
+ run_query=False,
+
+ error=ve.message
+ )
return self.render_template('explorer/query.html', vm)
@staticmethod
def get_instance_and_form(request, query_id):
query = get_object_or_404(Query, pk=query_id)
query.params = url_get_params(request)
- form = QueryForm(request.POST if len(request.POST) else None, instance=query)
+ form = QueryForm(request.POST if len(
+ request.POST) else None, instance=query)
return query, form
@@ -318,29 +396,36 @@ def query_viewmodel(request, query, title=None, form=None, message=None, run_que
rows = url_get_rows(request)
res = None
ql = None
+ lag_exists = False
+ replication_lag = None
if run_query:
try:
res, ql = query.execute_with_logging(request.user)
+ lag_exists, replication_lag = check_replication_lag()
except DatabaseError as e:
error = str(e)
has_valid_results = not error and res and run_query
ret = RequestContext(request, {
- 'tasks_enabled': app_settings.ENABLE_TASKS,
- 'params': query.available_params(),
- 'title': title,
- 'shared': query.shared,
- 'query': query,
- 'form': form,
- 'message': message,
- 'error': error,
- 'rows': rows,
- 'data': res.data[:rows] if has_valid_results else None,
- 'headers': res.headers if has_valid_results else None,
- 'total_rows': len(res.data) if has_valid_results else None,
- 'duration': res.duration if has_valid_results else None,
- 'has_stats': len([h for h in res.headers if h.summary]) if has_valid_results else False,
- 'dataUrl': reverse_lazy('query_csv', kwargs={'query_id': query.id}) if query.id else '',
- 'bucket': app_settings.S3_BUCKET,
- 'snapshots': query.snapshots if query.snapshot else [],
- 'ql_id': ql.id if ql else None})
- return ret
+ 'tasks_enabled': app_settings.ENABLE_TASKS,
+ 'params': query.available_params(),
+ 'title': title,
+ 'shared': query.shared,
+ 'query': query,
+ 'form': form,
+ 'message': message,
+ 'error': error,
+ 'rows': rows,
+ 'data': res.data[:rows] if has_valid_results else None,
+ 'headers': res.headers if has_valid_results else None,
+ 'total_rows': len(res.data) if has_valid_results else None,
+ 'duration': res.duration if has_valid_results else None,
+ 'has_stats': len([h for h in res.headers if h.summary]) if has_valid_results else False,
+ 'dataUrl': reverse_lazy('query_csv', kwargs={'query_id': query.id}) if query.id else '',
+ 'bucket': app_settings.S3_BUCKET,
+ 'snapshots': query.snapshots if query.snapshot else [],
+ 'ql_id': ql.id if ql else None,
+ 'lag_exists': lag_exists,
+ 'replication_lag': replication_lag,
+ }
+ )
+ return ret.flatten()
diff --git a/requirements.txt b/requirements.txt
index b4f0851b..893c8e70 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
-Django>=1.7.0
+Django>=1.11.17
factory-boy==2.6.0
mock==1.0.1
six==1.10.0
-sqlparse==0.1.18
-unicodecsv==0.14.1
\ No newline at end of file
+sqlparse>=0.4.0
+unicodecsv==0.14.1
+ago==0.0.93
diff --git a/setup.py b/setup.py
index 70dbb6cd..cc76a6be 100644
--- a/setup.py
+++ b/setup.py
@@ -12,8 +12,8 @@ def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
setup(
- name = "django-sql-explorer",
- version = __version__,
+ name = "django-sql-explorer-squad",
+ version = "0.9.25",
author = "Chris Clark",
author_email = "chris@untrod.com",
description = ("A pluggable app that allows users (admins) to execute SQL,"
@@ -37,10 +37,12 @@ def read(fname):
'Programming Language :: Python :: 3.4',
],
install_requires=[
- 'Django>=1.6.7',
+ 'Django>=1.11',
'sqlparse>=0.1.11',
'unicodecsv>=0.13.0',
'six>=1.10.0',
+ # custom deps
+ 'ago==0.0.93',
],
include_package_data=True,
zip_safe = False,
diff --git a/test_settings.py b/test_settings.py
new file mode 100644
index 00000000..af4262bc
--- /dev/null
+++ b/test_settings.py
@@ -0,0 +1,93 @@
+import os
+
+# import djcelery
+
+SECRET_KEY = "shhh"
+DEBUG = True
+STATIC_URL = "/static/"
+
+ALLOWED_HOSTS = ["0.0.0.0", "localhost", "127.0.0.1"]
+
+BASE_DIR = os.path.dirname(os.path.dirname(__file__))
+CSRF_COOKIE_HTTPONLY = False
+
+DATABASES = {
+ "default": {
+ "ENGINE": "django.db.backends.sqlite3",
+ "NAME": "tmp",
+ "TEST": {"NAME": "test_tmp"},
+ },
+ "alt": {
+ "ENGINE": "django.db.backends.sqlite3",
+ "NAME": "tmp2",
+ "TEST": {"NAME": "test_tmp2"},
+ },
+ "not_registered": {
+ "ENGINE": "django.db.backends.sqlite3",
+ "NAME": "tmp3",
+ "TEST": {"NAME": "test_tmp3"},
+ },
+}
+
+EXPLORER_CONNECTIONS = {
+ #'Postgres': 'postgres',
+ #'MySQL': 'mysql',
+ "SQLite": "default",
+ "Another": "alt",
+}
+EXPLORER_DEFAULT_CONNECTION = "default"
+
+ROOT_URLCONF = "explorer.tests.urls"
+
+TEMPLATES = [
+ {
+ "BACKEND": "django.template.backends.django.DjangoTemplates",
+ "DIRS": [],
+ "APP_DIRS": True,
+ "OPTIONS": {
+ "context_processors": [
+ "django.contrib.auth.context_processors.auth",
+ "django.contrib.messages.context_processors.messages",
+ "django.template.context_processors.static",
+ "django.template.context_processors.request",
+ ],
+ "debug": DEBUG,
+ },
+ },
+]
+
+INSTALLED_APPS = (
+ "django.contrib.auth",
+ "django.contrib.contenttypes",
+ "django.contrib.sessions",
+ "django.contrib.messages",
+ "django.contrib.staticfiles",
+ "django.contrib.admin",
+ "explorer",
+ # 'djcelery'
+)
+
+AUTHENTICATION_BACKENDS = ("django.contrib.auth.backends.ModelBackend",)
+
+
+MIDDLEWARE = [
+ "django.contrib.sessions.middleware.SessionMiddleware",
+ "django.middleware.common.CommonMiddleware",
+ "django.middleware.csrf.CsrfViewMiddleware",
+ "django.contrib.auth.middleware.AuthenticationMiddleware",
+ "django.contrib.messages.middleware.MessageMiddleware",
+]
+
+# TEST_RUNNER = 'djcelery.contrib.test_runner.CeleryTestSuiteRunner'
+
+# djcelery.setup_loader()
+CELERY_ALWAYS_EAGER = True
+BROKER_BACKEND = "memory"
+
+# Explorer-specific
+
+EXPLORER_TRANSFORMS = (("foo", '
{0}'), ("bar", "x: {0}"))
+
+EXPLORER_USER_QUERY_VIEWS = {}
+EXPLORER_TASKS_ENABLED = True
+EXPLORER_S3_BUCKET = "thisismybucket.therearemanylikeit.butthisoneismine"