checkpoint - not working

explorer/ee/db_connections/create_sqlite.py

@@ -1,7 +1,7 @@
 import os
 from io import BytesIO
 
-from explorer.ee.db_connections.type_infer import get_parser
+from explorer.ee.db_connections.type_infer import get_parser, is_sqlite
 from explorer.ee.db_connections.utils import pandas_to_sqlite, download_local_sqlite
 
 
@@ -12,6 +12,9 @@ def parse_to_sqlite(file, append=None) -> (BytesIO, str):
     local_path = f"{f_name}_tmp_local.db"
 
     if append:
+        if is_sqlite(file):
+            raise TypeError("Can't append a SQLite file to a SQLite file. Only CSV and JSON.")
+        # Get the sqlite file we are going to append to into the local filesystem
         download_local_sqlite(append, local_path)
 
     df_parser = get_parser(file)

explorer/ee/db_connections/views.py

@@ -1,5 +1,5 @@
 import logging
-from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView
+from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView, TemplateView
 from django.views import View
 from django.http import JsonResponse
 from django.urls import reverse_lazy
@@ -26,31 +26,40 @@ class UploadDbView(PermissionRequiredMixin, View):
 
     permission_required = "connections_permission"
 
-    def post(self, request):
+    def post(self, request):  # noqa
         file = request.FILES.get("file")
         if file:
+
+            append = request.POST.get("append")
+
             if file.size > EXPLORER_MAX_UPLOAD_SIZE:
                 friendly = EXPLORER_MAX_UPLOAD_SIZE / (1024 * 1024)
                 return JsonResponse({"error": f"File size exceeds the limit of {friendly} MB"}, status=400)
 
             try:
                 # The 'append' should be the s3 path of the sqlite DB to append this table to
-                f_bytes, f_name = parse_to_sqlite(file, append=request.POST.get("append"))
+                f_bytes, f_name = parse_to_sqlite(file, append)
             except ValueError as e:
                 logger.error(f"Error getting bytes for {file.name}: {e}")
                 return JsonResponse({"error": "File was not csv, json, or sqlite."}, status=400)
             except TypeError as e:
                 logger.error(f"Error parse {file.name}: {e}")
                 return JsonResponse({"error": "Error parsing file."}, status=400)
 
-            try:
+            if append:
+                s3_path = append
+            else:
                 s3_path = f"user_dbs/user_{request.user.id}/{f_name}"
+
+            try:
                 upload_sqlite(f_bytes, s3_path)
             except Exception as e:  # noqa
                 logger.exception(f"Exception while uploading file {f_name}: {e}")
                 return JsonResponse({"error": "Error while uploading file to S3."}, status=400)
 
-            create_connection_for_uploaded_sqlite(f_name, request.user.id, s3_path)
+            # If we are appending to an existing sqlite source, don't create a new DB connection record
+            if not append:
+                create_connection_for_uploaded_sqlite(f_name, request.user.id, s3_path)
             return JsonResponse({"success": True})
         else:
             return JsonResponse({"error": "No file provided"}, status=400)
@@ -86,6 +95,15 @@ class DatabaseConnectionCreateView(PermissionRequiredMixin, ExplorerContextMixin
     success_url = reverse_lazy("explorer_connections")
 
 
+class DatabaseConnectionUploadCreateView(TemplateView):
+    template_name = "connections/connection_upload.html"
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context["valid_connections"] = DatabaseConnection.objects.filter(engine=DatabaseConnection.SQLITE, host__isnull=False)
+        return context
+
+
 class DatabaseConnectionUpdateView(PermissionRequiredMixin, ExplorerContextMixin, UpdateView):
     permission_required = "connections_permission"
     model = DatabaseConnection

explorer/ee/urls.py

@@ -7,14 +7,16 @@
     DatabaseConnectionDetailView,
     DatabaseConnectionUpdateView,
     DatabaseConnectionDeleteView,
-    DatabaseConnectionValidateView
+    DatabaseConnectionValidateView,
+    DatabaseConnectionUploadCreateView,
 )
 
 ee_urls = [
     path("connections/", DatabaseConnectionsListView.as_view(), name="explorer_connections"),
     path("connections/upload/", UploadDbView.as_view(), name="explorer_upload"),
     path("connections/<int:pk>/", DatabaseConnectionDetailView.as_view(), name="explorer_connection_detail"),
     path("connections/new/", DatabaseConnectionCreateView.as_view(), name="explorer_connection_create"),
+    path("connections/create_upload/", DatabaseConnectionUploadCreateView.as_view(), name="explorer_upload_create"),
     path("connections/<int:pk>/edit/", DatabaseConnectionUpdateView.as_view(), name="explorer_connection_update"),
     path("connections/<int:pk>/delete/", DatabaseConnectionDeleteView.as_view(), name="explorer_connection_delete"),
     # There are two URLs here because the form can call validate from /connections/new/ or from /connections/<pk>/edit/

explorer/src/js/main.js

@@ -16,7 +16,7 @@ const route_initializers = {
     query_create:        () => import('./explorer').then(({ExplorerEditor}) => new ExplorerEditor('new')),
     explorer_playground: () => import('./explorer').then(({ExplorerEditor}) => new ExplorerEditor('new')),
     explorer_schema:     () => import('./schema').then(({setupSchema}) => setupSchema()),
-    explorer_connection_create:            () => import('./uploads').then(({setupUploads}) => setupUploads()),
+    explorer_upload_create:            () => import('./uploads').then(({setupUploads}) => setupUploads()),
     explorer_connection_update:            () => import('./uploads').then(({setupUploads}) => setupUploads())
 };
 

explorer/src/js/uploads.js

@@ -45,6 +45,12 @@ export function setupUploads() {
         let formData = new FormData();
         formData.append('file', file);
 
+        let appendElem = document.getElementById('append');
+        let appendValue = appendElem.value;
+        if (appendValue) {
+            formData.append('append', appendValue);
+        }
+
         let xhr = new XMLHttpRequest();
         xhr.open('POST', '../upload/', true);
         xhr.setRequestHeader('X-CSRFToken', getCsrfToken());
@@ -63,9 +69,8 @@ export function setupUploads() {
 
         xhr.onload = function() {
             if (xhr.status === 200) {
-                let fileName = file.name;
-                let fileNameWithoutExt = fileName.substring(0, fileName.lastIndexOf('.')) || fileName;
-                window.location.href = `../?highlight=${encodeURIComponent(fileNameWithoutExt)}`;
+                let highlightValue = appendValue ? appendElem.options[appendElem.selectedIndex].text : file.name.substring(0, file.name.lastIndexOf('.')) || file.name;
+                 window.location.href = `../?highlight=${encodeURIComponent(highlightValue)}`;
             } else {
                 console.error('Error:', xhr.response);
                 uploadStatus.innerHTML = xhr.response;
@@ -80,25 +85,28 @@ export function setupUploads() {
         xhr.send(formData);
     }
 
-    document.getElementById("test-connection-btn").addEventListener("click", function() {
-        var form = document.getElementById("db-connection-form");
-        var formData = new FormData(form);
-
-        fetch("../validate/", {
-            method: "POST",
-            body: formData,
-            headers: {
-                "X-CSRFToken": getCsrfToken()
-            }
-        })
-        .then(response => response.json())
-        .then(data => {
-            if (data.success) {
-                alert("Connection successful!");
-            } else {
-                alert("Connection failed: " + data.error);
-            }
-        })
-        .catch(error => console.error("Error:", error));
-    });
+    let testConnBtn = document.getElementById("test-connection-btn");
+    if (testConnBtn) {
+        testConnBtn.addEventListener("click", function() {
+            let form = document.getElementById("db-connection-form");
+            let formData = new FormData(form);
+
+            fetch("../validate/", {
+                method: "POST",
+                body: formData,
+                headers: {
+                    "X-CSRFToken": getCsrfToken()
+                }
+            })
+            .then(response => response.json())
+            .then(data => {
+                if (data.success) {
+                    alert("Connection successful!");
+                } else {
+                    alert("Connection failed: " + data.error);
+                }
+            })
+            .catch(error => console.error("Error:", error));
+        });
+    }
 }

explorer/templates/connections/connection_upload.html

@@ -0,0 +1,30 @@
+{% extends 'explorer/base.html' %}
+{% block sql_explorer_content %}
+<div class="container mt-5">
+    <div class="pt-3">
+        <h4>Upload a file</h4>
+        <p>Supports .csv, .json, .db, and .sqlite files. JSON files with one JSON document per line are also supported. CSV/JSON data will be parsed and converted to SQLite. SQLite databases must <i>not</i> be password protected.</p>
+        <p>You can upload a file as a new data source, or append it to an existing uploaded source. When appending, if a table with the same name as the uploaded file already exists, it will be dropped and replaced with the uploaded data.</p>
+
+        <form id="upload-form">
+            <div class="form-group">
+                <label for="append">Append to existing connection:</label>
+                <select id="append" name="append" class="form-control">
+                    <option value="" selected>None</option>
+                    {% for connection in valid_connections %}
+                        <option value="{{ connection.host }}">{{ connection.alias }}</option>
+                    {% endfor %}
+                </select>
+            </div>
+            <div id="drop-area" class="p-3 mb-4 bg-light border rounded" style="cursor: pointer">
+                <p class="lead mb-0">Drag and drop, or click to upload .csv, .json, .db, .sqlite.</p>
+                <input type="file" id="fileElem" style="display:none" accept=".db,.csv,.sqlite,.json">
+                <div class="progress mt-3" style="height: 20px;">
+                    <div id="progress-bar" class="progress-bar" role="progressbar" style="width: 0;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
+                </div>
+                <p id="upload-status" class="mt-2"></p>
+            </div>
+        </form>
+    </div>
+</div>
+{% endblock %}

explorer/templates/connections/connections.html

@@ -6,6 +6,7 @@
     <h3>Connections</h3>
     <div class="mt-3">
         <a href="{% url 'explorer_connection_create' %}" class="btn btn-primary mb-3">Add New Connection</a>
+        <a href="{% url 'explorer_upload_create' %}" class="btn btn-primary mb-3">Upload File</a>
         <table class="table table-striped" id="connections_table">
             <thead>
                 <tr>

explorer/templates/connections/database_connection_form.html

@@ -2,27 +2,9 @@
 {% block sql_explorer_content %}
 <div class="container mt-5">
     <h2>{% if object %}Edit{% else %}Create New{% endif %} Connection</h2>
-    {% if not object %}
-        <p>SQL Explorer supports two connection methods; uploading a file, or configuring a connection to an existing database.</p>
-    {% endif %}
 {% if object.is_upload %}
     <span class="text-danger">The source of this connection is an uploaded file. In all likelihood you should not be editing it.</span>
 {% endif %}
-    {% if not object and user_uploads_enabled %}
-        <div class="pt-3">
-            <h4>Upload a file</h4>
-            <p>Supports .csv, .json, .db, and .sqlite files. JSON files with one JSON document per line are also supported. CSV/JSON data will be parsed and converted to SQLite. SQLite databases must <i>not</i> be password protected.</p>
-            <div id="drop-area" class="p-3 mb-4 bg-light border rounded" style="cursor: pointer">
-                <p class="lead mb-0">Drag and drop, or click to upload .csv, .json, .db, .sqlite.</p>
-                <input type="file" id="fileElem" style="display:none" accept=".db,.csv,.sqlite,.json">
-                <div class="progress mt-3" style="height: 20px;">
-                    <div id="progress-bar" class="progress-bar" role="progressbar" style="width: 0;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
-                </div>
-                <p id="upload-status" class="mt-2"></p>
-            </div>
-        </div>
-    {% endif %}
-    <h4>Configure a connection to an existing database.</h4>
     <form method="post" id="db-connection-form">
         {% csrf_token %}
         <div>

explorer/templates/explorer/schema.html

@@ -6,7 +6,6 @@
 <div class="schema-wrapper">
     <h4>{% translate "Schema" %}</h4>
     <div id="schema-contents">
-    {% if m|length > 1 %}
         <p><input class="search form-control" placeholder="{% translate "Search Tables" %}" /></p>
         <div class="row">
             <div class="col">
@@ -20,7 +19,6 @@ <h4>{% translate "Schema" %}</h4>
                 </a>
             </div>
         </div>
-    {% endif %}
         <div class="mt-3">
             <ul class="list">
                 {% for m in schema %}