basic anonymous usage telemetry (#597)

#597

.gitignore

@@ -20,3 +20,4 @@ explorer/static/
 # Sphinx documentation
 docs/_build/
 .env
+tst

HISTORY.rst

@@ -5,7 +5,12 @@ Change Log
 This document records all notable changes to `django-sql-explorer <https://github.com/chrisclark/django-sql-explorer>`_.
 This project adheres to `Semantic Versioning <https://semver.org/>`_.
 
-`4.0.0`_ (2024-02-06)
+`4.1.0`_ (TBD)
+===========================
+* Anonymous usage telemetry. Can be disabled by setting EXPLORER_ENABLE_ANONYMOUS_STATS to False
+* `#594`_: Eliminate <script> tags to prevent potential Content Security Policy issues
+
+`4.0.2`_ (2024-02-06)
 ===========================
 * Add support for Django 5.0. Drop support for Python < 3.10.
 * Basic code completion in the editor!

README.rst

@@ -17,7 +17,9 @@
 SQL Explorer
 ============
 
-`Documentation <https://django-sql-explorer.readthedocs.io/en/latest/>`_
+`Official Website <https://www.sqlexplorer.io/>`_
+
+`Full Documentation <https://django-sql-explorer.readthedocs.io/en/late st/>`_
 
 SQL Explorer aims to make the flow of data between people fast,
 simple, and confusion-free. It is a Django-based application that you

docs/dependencies.rst

@@ -7,13 +7,15 @@ minimum.
 Python
 ------
 
-=========================================================== ======= ================
-Name                                                        Version License
-=========================================================== ======= ================
-`sqlparse <https://github.com/andialbrecht/sqlparse/>`_     0.4.0   BSD
-=========================================================== ======= ================
+============================================================ ======= ================
+Name                                                         Version License
+============================================================ ======= ================
+`sqlparse <https://github.com/andialbrecht/sqlparse/>`_      0.4.0   BSD
+`requests <https://requests.readthedocs.io/en/latest/>`_     2.2.0   Apache 2.0
+============================================================ ======= ================
 
 - sqlparse is used for SQL formatting
+- requests is used for anonymous usage tracking
 
 **Python - Optional Dependencies**
 
@@ -42,30 +44,3 @@ The bundle for the SQL editor is fairly large at ~400kb, due primarily to CodeMi
 
 The built front-end files are distributed in the PyPi release (and will be found by collectstatic). Instructions for building the front-end files are in :doc:`install`.
 
-Tests
------
-
-Factory Boy is needed if you'd like to run the tests, which can you do
-easily:
-
-``python manage.py test``
-
-and with coverage:
-
-``coverage run --source='.' manage.py test``
-
-then:
-
-``coverage report``
-
-...97%! Huzzah!
-
-Running Locally
----------------
-
-Included is a test_project that you can use to kick the tires. Just
-create a new virtualenv, cd into ``test_project`` and run ``start.sh`` (or
-walk through the steps yourself) to get a test instance of the app up
-and running.
-
-You can now navigate to 127.0.0.1:8000/ and begin exploring!

docs/development.rst

@@ -0,0 +1,63 @@
+Running Locally (quick start)
+-----------------------------
+
+Included is a test_project that you can use to kick the tires. Just
+create a new virtualenv, cd into ``test_project`` and run ``start.sh`` (or
+walk through the steps yourself) to get a test instance of the app up
+and running.
+
+You can now navigate to 127.0.0.1:8000/ and begin exploring!
+
+Installing From Source
+----------------------
+
+If you are installing SQL Explorer from source (by cloning the repository),
+you may want to first look at simply running test_project/start.sh.
+
+If you want to install into an existing project, you can do so by following
+the install instructions, and then additionally building the front-end dependencies.
+
+After cloning, simply run:
+
+::
+
+    nvm install
+    nvm use
+    npm install
+    npm run build
+
+The front-end assets will be built and placed in the /static/ folder
+and collected properly by your Django installation during the `collect static`
+phase. Copy the /explorer directory into site-packages and you're ready to go.
+
+Tests
+-----
+
+Factory Boy is needed if you'd like to run the tests, which can you do
+easily:
+
+``python manage.py test --settings=tests.settings``
+
+and with coverage:
+
+``coverage run --source='.' manage.py test --settings=tests.settings``
+``coverage combine``
+``coverage report``
+
+Running Celery
+--------------
+
+To run tests with Celery enabled, you will need to install Redis and Celery.
+::
+
+    brew install redis
+    pip install celery
+    pip install redis
+
+Then run the redis server and the celery worker. A good way of doing it is:
+::
+
+    screen -d -S 'redis' -m redis-server
+    screen -d -S 'celery' -m celery -A test_project worker
+
+Finally, set ``EXPLORER_TASKS_ENABLED`` to True in tests.settings and run the tests.

docs/index.rst

@@ -16,6 +16,7 @@ Django SQL Explorer
 
    features.rst
    install.rst
+   development.rst
    settings.rst
    dependencies.rst
    history.rst

docs/install.rst

@@ -102,29 +102,10 @@ settings.py to enable these features.
 Installing From Source
 ----------------------
 
-If you are installing SQL Explorer from source (by cloning the repository),
-you may want to first look at simply running test_project/start.sh.
-
-If you want to install it into an existing project, you can do so by following
-the instructions above, and additionally building the front-end dependencies.
-
-After cloning, simply run:
-
-::
-
-    nvm install
-    nvm use
-    npm install
-    npm run build
-
-The front-end assets will be built and placed in the /static/ folder
-and collected properly by your Django installation during the `collect static`
-phase. Copy the /explorer directory into site-packages and you're ready to go.
-
-And frankly, as long as you have a reasonably modern version of Node and NPM
-installed, you can probably skip the nvm steps.
-
 Because the front-end assets must be built, installing SQL Explorer via pip
 from github is not supported. The package will be installed, but the front-end
 assets will be missing and will not be able to be built, as the necessary
 configuration files are not included when github builds the wheel for pip.
+
+To run from source, clone the repository and follow the :doc:`development`
+instructions.

docs/settings.rst

@@ -319,7 +319,7 @@ The export buttons to use. Default includes Excel, so xlsxwriter from ``requirem
 
 
 Unsafe rendering
-*****************************
+****************
 
 Disable auto escaping for rendering values from the database. Be wary of XSS attacks if querying unknown data.
 
@@ -337,3 +337,13 @@ but a dotted path to a python view can be used
 .. code-block:: python
 
    EXPLORER_NO_PERMISSION_VIEW = 'explorer.views.auth.safe_login_view_wrapper'
+
+Anonymous Usage Stat Collection
+*******************************
+
+By default, anonymous usage statistics are collected. To disable this, set the following setting to False.
+You can see what is being collected in tracker.py.
+
+.. code-block:: python
+
+    EXPLORER_ENABLE_ANONYMOUS_STATS = False

explorer/app_settings.py

@@ -104,7 +104,7 @@
 EXPLORER_TOKEN_AUTH_ENABLED = lambda: getattr(  # noqa
     settings, "EXPLORER_TOKEN_AUTH_ENABLED", False
 )
-EXPLORER_NO_PERMISSION_VIEW = lambda: locate(# noqa
+EXPLORER_NO_PERMISSION_VIEW = lambda: locate(  # noqa
     getattr(
         settings,
         "EXPLORER_NO_PERMISSION_VIEW",
@@ -133,6 +133,9 @@
 
 EXPLORER_SHOW_SQL_BY_DEFAULT = getattr(settings, "EXPLORER_SHOW_SQL_BY_DEFAULT", True)
 
+EXPLORER_ENABLE_ANONYMOUS_STATS = getattr(settings, "EXPLORER_ENABLE_ANONYMOUS_STATS", True)
+EXPLORER_COLLECT_ENDPOINT_URL = "https://collect.sqlexplorer.io/stat"
+
 # If set to True will autorun queries when viewed which is the historical behavior
 # Default to True if not set in order to be backwards compatible
 # If set to False will not autorun queries containing parameters when viewed

explorer/apps.py

@@ -1,6 +1,7 @@
 from django.apps import AppConfig
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connections as djcs
+from django.db.utils import OperationalError
 from django.utils.translation import gettext_lazy as _
 
 
@@ -14,6 +15,7 @@ def ready(self):
         from explorer.schema import build_async_schemas
         _validate_connections()
         build_async_schemas()
+        track_summary_stats()
 
 
 def _get_default():
@@ -42,3 +44,25 @@ def _validate_connections():
                 f"EXPLORER_CONNECTIONS contains ({name}, {conn_name}), "
                 f"but {conn_name} is not a valid Django DB connection."
             )
+
+
+def track_summary_stats():
+    from explorer.tracker import Stat, StatNames
+    from explorer.tracker import gather_summary_stats
+    from explorer.models import Query
+
+    # Django doesn't actually have a way of running code on application initialization, so we have come up with this.
+    # The app.ready() method (the call site for this function) is invoked *before* any migrations are run. So if were
+    # to just call this function in ready(), without the try: block, then it would always fail the very first time
+    # Django runs (and e.g. in test runs) because no tables have yet been created. The intuitive way to handle this with
+    # Django would be to tie into the post_migrate signal in ready() and run this function on post_migrate. But that
+    # doesn't work because that signal is only called if indeed a migrations has been applied. If the app restarts and
+    # there are no new migrations, the signal never fires. So instead we check if the Query table exists, and if it
+    # does, we're good to gather stats.
+    try:
+        Query.objects.first()
+    except OperationalError:
+        return
+    else:
+        payload = gather_summary_stats()
+        Stat(StatNames.STARTUP_STATS, payload).track()

explorer/models.py

@@ -8,6 +8,7 @@
 from django.utils.translation import gettext_lazy as _
 
 from explorer import app_settings
+from explorer.tracker import Stat, StatNames
 from explorer.utils import (
     extract_params, get_params_for_url, get_s3_bucket, get_valid_connection, passes_blacklist, s3_url,
     shared_dict_update, swap_params,
@@ -108,6 +109,8 @@ def execute_with_logging(self, executing_user):
             raise e
         ql.duration = ret.duration
         ql.save()
+        Stat(StatNames.QUERY_RUN,
+             {"sql_len": len(ql.sql), "duration": ql.duration}).track()
         return ret, ql
 
     def execute(self):

explorer/templates/explorer/base.html

@@ -72,7 +72,7 @@ <h2>This is easy to fix, I promise!</h2>
     <div class="container">
         <footer class="py-3 my-4">
             <p class="text-center text-body-secondary border-top pt-3">
-                Powered by <a href="https://www.github.com/chrisclark/django-sql-explorer/">django-sql-explorer</a>. Rendered at {% now "SHORT_DATETIME_FORMAT" %}
+                Powered by <a href="https://www.sqlexplorer.io/">SQL Explorer</a>. Rendered at {% now "SHORT_DATETIME_FORMAT" %}
             </p>
         </footer>
     </div>

explorer/tests/settings.py

@@ -0,0 +1,30 @@
+from test_project.settings import *  # noqa
+
+EXPLORER_ENABLE_ANONYMOUS_STATS = False
+EXPLORER_TASKS_ENABLED = False  # set to true to test async tasks
+CELERY_BROKER_URL = "redis://localhost:6379/0"
+CELERY_TASK_ALWAYS_EAGER = True
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": "tst",
+        "TEST": {
+            "NAME": "tst"
+        }
+    },
+    "alt": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": "tst2",
+        "TEST": {
+            "NAME": "tst2"
+        }
+    },
+    "not_registered": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": "tst3",
+        "TEST": {
+            "NAME": "tst3"
+        }
+    }
+}

explorer/tests/test_tasks.py

@@ -85,7 +85,7 @@ def test_truncating_querylogs(self):
     @unittest.skipIf(not app_settings.ENABLE_TASKS, "tasks not enabled")
     @patch("explorer.schema.build_schema_info")
     def test_build_schema_cache_async(self, mocked_build):
-        mocked_build.return_value = ["list_of_tuples"]
+        mocked_build.return_value = [("table", [("column", "Integer")]),]
         schema = build_schema_cache_async(CONN)
         assert mocked_build.called
-        self.assertEqual(schema, ["list_of_tuples"])
+        self.assertEqual(schema, [("table", [("column", "Integer")]),])

explorer/tests/test_tracker.py

@@ -0,0 +1,18 @@
+from django.test import TestCase
+from explorer.tracker import instance_identifier, gather_summary_stats
+
+
+class TestTracker(TestCase):
+
+    def test_instance_identifier(self):
+        v = instance_identifier()
+
+        # The SHA-256 hash produces a fixed-length output of 256 bits.
+        # When represented as a hexadecimal string, each byte (8 bits) is
+        # represented by 2 hex chars. 256/8*2 = 64
+        self.assertEqual(len(v), 64)
+
+    def test_gather_summary_stats(self):
+        res = gather_summary_stats()
+        self.assertEqual(res["total_query_count"], 0)
+        self.assertEqual(res["default_database"], "sqlite")