addition of new tables for async api db

.gitignore

@@ -5,9 +5,10 @@
 /dist
 *.egg-info
 .DS_Store
+tmp
 /build
 *#
 *~
 .coverage
 /htmlcov/
-*.orig
+*.orig

Dockerfile

@@ -0,0 +1,21 @@
+# Use an official Python runtime as a parent image
+FROM python:3.5-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy the requirements file into the container at /app
+COPY requirements.txt /app/
+COPY optional-requirements.txt /app/
+
+# Install any needed packages specified in requirements.txt
+RUN pip install Django==1.11.17
+RUN pip install -r requirements.txt
+RUN pip install -r optional-requirements.txt
+
+# Copy the entire Django project directory into the container at /app
+COPY . /app/

docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3'
+services:
+  db:
+    image: postgres:13
+    environment:
+      POSTGRES_DB: mydb
+      POSTGRES_USER: myuser
+      POSTGRES_PASSWORD: mypassword
+
+  web:
+    build: .
+    command: python manage.py runserver 0.0.0.0:8000
+    volumes:
+      - .:/app
+    ports:
+      - "8000:8000"
+    depends_on:
+      - db
+    environment:
+      DEBUG: "True"  # Set to "False" in production
+      DJANGO_DB_HOST: db
+      DJANGO_DB_PORT: 5432
+      DJANGO_DB_NAME: mydb
+      DJANGO_DB_USER: myuser
+      DJANGO_DB_PASSWORD: mypassword
+    links:
+      - db

explorer/__init__.py

@@ -1,7 +1,7 @@
 __version_info__ = {
     'major': 0,
     'minor': 9,
-    'micro': 2,
+    'micro': 23,
     'releaselevel': 'final',
     'serial': 0
 }

explorer/admin.py

@@ -1,3 +1,4 @@
+
 from django.contrib import admin
 from explorer.models import Query
 from explorer.actions import generate_report_action
@@ -7,7 +8,8 @@ class QueryAdmin(admin.ModelAdmin):
     list_display = ('title', 'description', 'created_by_user',)
     list_filter = ('title',)
     raw_id_fields = ('created_by_user',)
-    
+
     actions = [generate_report_action()]
 
+
 admin.site.register(Query, QueryAdmin)

explorer/app_settings.py

@@ -2,28 +2,49 @@
 
 # Required
 EXPLORER_CONNECTION_NAME = getattr(settings, 'EXPLORER_CONNECTION_NAME', None)
+EXPLORER_CONNECTION_PII_NAME = getattr(
+    settings, 'EXPLORER_CONNECTION_PII_NAME', None)
+EXPLORER_CONNECTION_ASYNC_API_DB_NAME = getattr(
+    settings, 'EXPLORER_CONNECTION_ASYNC_API_DB_NAME', None)
+EXPLORER_MASTER_DB_CONNECTION_NAME = getattr(settings, 'EXPLORER_MASTER_DB_CONNECTION', None)
 
 # Change the behavior of explorer
-EXPLORER_SQL_BLACKLIST = getattr(settings, 'EXPLORER_SQL_BLACKLIST', ('ALTER', 'RENAME ', 'DROP', 'TRUNCATE', 'INSERT INTO', 'UPDATE', 'REPLACE', 'DELETE', 'CREATE TABLE', 'SCHEMA', 'GRANT', 'OWNER TO'))
-EXPLORER_SQL_WHITELIST = getattr(settings, 'EXPLORER_SQL_WHITELIST', ('CREATED', 'DELETED', 'REGEXP_REPLACE'))
+EXPLORER_SQL_BLACKLIST = getattr(settings, 'EXPLORER_SQL_BLACKLIST', ('ALTER', 'RENAME ', 'DROP', 'TRUNCATE',
+                                 'INSERT INTO', 'UPDATE', 'REPLACE', 'DELETE', 'CREATE TABLE', 'SCHEMA', 'GRANT', 'OWNER TO'))
+EXPLORER_SQL_WHITELIST = getattr(
+    settings, 'EXPLORER_SQL_WHITELIST', ('CREATED', 'DELETED', 'REGEXP_REPLACE'))
+TABLE_NAMES_FOR_PII_MASKING = getattr(
+    settings, 'TABLE_NAMES_FOR_PII_MASKING', None)
 EXPLORER_DEFAULT_ROWS = getattr(settings, 'EXPLORER_DEFAULT_ROWS', 1000)
-EXPLORER_SCHEMA_EXCLUDE_APPS = getattr(settings, 'EXPLORER_SCHEMA_EXCLUDE_APPS', ('django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.admin'))
+EXPLORER_SCHEMA_EXCLUDE_APPS = getattr(settings, 'EXPLORER_SCHEMA_EXCLUDE_APPS', (
+    'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.admin'))
 EXPLORER_TRANSFORMS = getattr(settings, 'EXPLORER_TRANSFORMS', [])
-EXPLORER_PERMISSION_VIEW = getattr(settings, 'EXPLORER_PERMISSION_VIEW', lambda u: u.is_staff)
-EXPLORER_PERMISSION_CHANGE = getattr(settings, 'EXPLORER_PERMISSION_CHANGE', lambda u: u.is_staff)
-EXPLORER_RECENT_QUERY_COUNT = getattr(settings, 'EXPLORER_RECENT_QUERY_COUNT', 10)
+EXPLORER_PERMISSION_VIEW = getattr(
+    settings, 'EXPLORER_PERMISSION_VIEW', lambda u: u.is_staff)
+EXPLORER_PERMISSION_CHANGE = getattr(
+    settings, 'EXPLORER_PERMISSION_CHANGE', lambda u: u.is_staff)
+EXPLORER_RECENT_QUERY_COUNT = getattr(
+    settings, 'EXPLORER_RECENT_QUERY_COUNT', 10)
 CSV_DELIMETER = getattr(settings, "EXPLORER_CSV_DELIMETER", ",")
 
 # API access
 EXPLORER_TOKEN = getattr(settings, 'EXPLORER_TOKEN', 'CHANGEME')
 # These are callable to aid testability by dodging the settings cache.
 # There is surely a better pattern for this, but this'll hold for now.
-EXPLORER_GET_USER_QUERY_VIEWS = lambda: getattr(settings, 'EXPLORER_USER_QUERY_VIEWS', {})
-EXPLORER_TOKEN_AUTH_ENABLED = lambda: getattr(settings, 'EXPLORER_TOKEN_AUTH_ENABLED', False)
+
+
+def EXPLORER_GET_USER_QUERY_VIEWS(): return getattr(
+    settings, 'EXPLORER_USER_QUERY_VIEWS', {})
+
+
+def EXPLORER_TOKEN_AUTH_ENABLED(): return getattr(
+    settings, 'EXPLORER_TOKEN_AUTH_ENABLED', False)
+
 
 # Async task related. Note that the EMAIL_HOST settings must be set up for email to work.
 ENABLE_TASKS = getattr(settings, "EXPLORER_TASKS_ENABLED", False)
 S3_ACCESS_KEY = getattr(settings, "EXPLORER_S3_ACCESS_KEY", None)
 S3_SECRET_KEY = getattr(settings, "EXPLORER_S3_SECRET_KEY", None)
 S3_BUCKET = getattr(settings, "EXPLORER_S3_BUCKET", None)
-FROM_EMAIL = getattr(settings, 'EXPLORER_FROM_EMAIL', '[email protected]')
+FROM_EMAIL = getattr(settings, 'EXPLORER_FROM_EMAIL',
+                     '[email protected]')

explorer/constants.py

@@ -0,0 +1,15 @@
+PII_MASKING_PATTERN_REPLACEMENT_DICT = {
+    r"(?:\+?\d{1,3}|0)?([6-9]\d{9})\b": "XXXXXXXXXXX", # For phone number
+    r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b": "[email protected]", # For email
+}
+
+TYPE_CODE_FOR_JSON = 3802
+TYPE_CODE_FOR_TEXT = 25
+TYPE_CODE_FOR_CHAR = 1043
+
+PLAYER_PHONE_NUMBER_MASKING_TYPE_CODES = [TYPE_CODE_FOR_CHAR]
+
+ALLOW_PHONE_NUMBER_MASKING_GROUP_ID = 10439
+
+PATTERN_FOR_FINDING_PHONE_NUMBER = r"\+?\d{0,3}?([6-9]\d{9})(?:_\w+)?\b"
+PATTERN_FOR_FINDING_EMAIL = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,7}\b"

explorer/forms.py

@@ -2,8 +2,12 @@
 from django.forms.widgets import CheckboxInput
 from explorer.models import Query, MSG_FAILED_BLACKLIST
 from django.db import DatabaseError
+import logging
+import re
+def _(x): return x
 
-_ = lambda x: x
+
+logger = logging.getLogger(__name__)
 
 
 class SqlField(Field):
@@ -14,20 +18,27 @@ def validate(self, value):
 
         :param value: The SQL for this Query model.
         """
-
+        super().validate(value)
         query = Query(sql=value)
 
         passes_blacklist, failing_words = query.passes_blacklist()
 
-        error = MSG_FAILED_BLACKLIST % ', '.join(failing_words) if not passes_blacklist else None
+        error = MSG_FAILED_BLACKLIST % ', '.join(
+            failing_words) if not passes_blacklist else None
 
         if not error and not query.available_params():
             try:
                 query.execute_query_only()
             except DatabaseError as e:
-                error = str(e)
+
+                logger.info("error executing query: %s", e)
+                if (re.search("permission denied for table", str(e))):
+                    error = None
+                else:
+                    error = e
 
         if error:
+
             raise ValidationError(
                 _(error),
                 code="InvalidSql"
@@ -54,4 +65,4 @@ def created_by_user_id(self):
 
     class Meta:
         model = Query
-        fields = ['title', 'sql', 'description', 'created_by_user', 'snapshot']
+        fields = ['title', 'sql', 'description', 'created_by_user', 'snapshot']

explorer/migrations/0001_initial.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 from django.db import models, migrations
 import django.db.models.deletion
@@ -14,34 +13,74 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.CreateModel(
-            name='Query',
+            name="Query",
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('title', models.CharField(max_length=255)),
-                ('sql', models.TextField()),
-                ('description', models.TextField(null=True, blank=True)),
-                ('created_at', models.DateTimeField(auto_now_add=True)),
-                ('last_run_date', models.DateTimeField(auto_now=True)),
-                ('created_by_user', models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, null=True)),
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID",
+                        serialize=False,
+                        auto_created=True,
+                        primary_key=True,
+                    ),
+                ),
+                ("title", models.CharField(max_length=255)),
+                ("sql", models.TextField()),
+                ("description", models.TextField(null=True, blank=True)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("last_run_date", models.DateTimeField(auto_now=True)),
+                (
+                    "created_by_user",
+                    models.ForeignKey(
+                        blank=True,
+                        to=settings.AUTH_USER_MODEL,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                    ),
+                ),
             ],
             options={
-                'ordering': ['title'],
-                'verbose_name_plural': 'Queries',
+                "ordering": ["title"],
+                "verbose_name_plural": "Queries",
             },
             bases=(models.Model,),
         ),
         migrations.CreateModel(
-            name='QueryLog',
+            name="QueryLog",
             fields=[
-                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('sql', models.TextField()),
-                ('is_playground', models.BooleanField(default=False)),
-                ('run_at', models.DateTimeField(auto_now_add=True)),
-                ('query', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, blank=True, to='explorer.Query', null=True)),
-                ('run_by_user', models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, null=True)),
+                (
+                    "id",
+                    models.AutoField(
+                        verbose_name="ID",
+                        serialize=False,
+                        auto_created=True,
+                        primary_key=True,
+                    ),
+                ),
+                ("sql", models.TextField()),
+                ("is_playground", models.BooleanField(default=False)),
+                ("run_at", models.DateTimeField(auto_now_add=True)),
+                (
+                    "query",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        blank=True,
+                        to="explorer.Query",
+                        null=True,
+                    ),
+                ),
+                (
+                    "run_by_user",
+                    models.ForeignKey(
+                        blank=True,
+                        to=settings.AUTH_USER_MODEL,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                    ),
+                ),
             ],
             options={
-                'ordering': ['-run_at'],
+                "ordering": ["-run_at"],
             },
             bases=(models.Model,),
         ),

explorer/migrations/0002_auto_20150501_1515.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 from django.db import models, migrations
 

explorer/migrations/0003_query_snapshot.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 from django.db import models, migrations
 

explorer/migrations/0004_querylog_duration.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 from django.db import migrations, models
 

explorer/migrations/0005_querychangelog.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9 on 2022-03-25 05:46
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('explorer', '0004_querylog_duration'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='QueryChangeLog',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('old_sql', models.TextField(blank=True, null=True)),
+                ('new_sql', models.TextField(blank=True, null=True)),
+                ('run_at', models.DateTimeField(auto_now_add=True)),
+                ('query', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='explorer.Query')),
+                ('run_by_user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['-run_at'],
+            },
+        ),
+    ]

explorer/migrations/0006_auto_20230207_0103.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('explorer', '0005_querychangelog'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='query',
+            name='snapshot',
+            field=models.BooleanField(default=False, help_text='Include in snapshot task (if enabled)'),
+        ),
+    ]