Fast json (#5082)

* Add simdjson to deps
* Fix Hash editor layour in "new" state
* Allow adding key values from file
* Improve formatters aтd json handling:
- Minify JSON with simdjson
- Format JSON with cpp code without validation
- Memorize selected formatters for each key
- Remove json-tools.js
- Simplify code in MultilineEditor

Fix #5061
Fix #4964
Fix #4859

Author: uglide

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "3rdparty/lz4"]
 	path = 3rdparty/lz4
 	url = https://github.com/lz4/lz4.git
+[submodule "3rdparty/simdjson"]
+	path = 3rdparty/simdjson
+	url = https://github.com/simdjson/simdjson.git

3rdparty/3rdparty.pri

@@ -33,6 +33,13 @@ include($$PWD/pyotherside.pri)
 LZ4DIR = $$PWD/lz4/
 INCLUDEPATH += $$LZ4DIR/lib
 
+#SIMDJSON
+SIMDJSONDIR = $$PWD/simdjson/singleheader
+INCLUDEPATH += $$SIMDJSONDIR/
+HEADERS += $$SIMDJSONDIR/simdjson.h
+SOURCES += $$SIMDJSONDIR/simdjson.cpp
+
+
 win32* {
     ZLIBDIR = $$PWD/zlib-msvc14-x64.1.2.11.7795/build/native    
     INCLUDEPATH += $$ZLIBDIR/include

3rdparty/simdjson

@@ -4,6 +4,7 @@
 #include <qredisclient/utils/text.h>
 
 #include <QObject>
+#include <QFile>
 
 #include "hashkey.h"
 #include "listkey.h"
@@ -119,7 +120,20 @@ void KeyFactory::submitNewKeyRequest(NewKeyRequest r) {
           return onRowAdded(testResp);
         }
 
-        result->addRow(r.value(), onRowAdded);
+        auto val = r.value();
+
+        if (!r.valueFilePath().isEmpty() && QFile::exists(r.valueFilePath())) {
+            QFile valueFile(r.valueFilePath());
+
+            if (!valueFile.open(QIODevice::ReadOnly)) {
+                return onRowAdded(QCoreApplication::translate(
+                                      "RDM", "Cannot open file with key value"));
+            }
+
+            val["value"] = valueFile.readAll();            
+        }
+
+        result->addRow(val, onRowAdded);
       },
       onRowAdded);
 }

src/app/models/key-models/keyfactory.cpp

@@ -11,14 +11,16 @@ class NewKeyRequest {
   Q_PROPERTY(QString keyName READ keyName WRITE setKeyName)
   Q_PROPERTY(QString keyType READ keyType WRITE setKeyType)
   Q_PROPERTY(QVariantMap value READ value WRITE setValue)
+  Q_PROPERTY(QString valueFilePath READ valueFilePath WRITE setValueFilePath)
 
  public:
   NewKeyRequest(QSharedPointer<RedisClient::Connection> connection, int dbIndex,
                 std::function<void()> callback, QString keyPrefix = QString())
       : m_connection(connection),
         m_dbIndex(dbIndex),
         m_callback(callback),
-        m_keyName(keyPrefix) {}
+        m_keyName(keyPrefix),
+        m_valueFilePath(QString()){}
 
   NewKeyRequest() {}
 
@@ -42,6 +44,10 @@ class NewKeyRequest {
 
   void setValue(const QVariantMap& v) { m_value = v; }
 
+  QString valueFilePath() const { return m_valueFilePath; }
+
+  void setValueFilePath(const QString& path) { m_valueFilePath = path; }
+
   QSharedPointer<RedisClient::Connection> connection() { return m_connection; }
 
   void callback() const {
@@ -55,6 +61,7 @@ class NewKeyRequest {
   QString m_keyName;
   QString m_keyType;
   QVariantMap m_value;
+  QString m_valueFilePath;
 };
 
 Q_DECLARE_METATYPE(NewKeyRequest)

src/app/models/key-models/newkeyrequest.h

@@ -13,6 +13,8 @@
 #include <QtConcurrent>
 #include <QUrl>
 
+#include <simdjson.h>
+
 #include "apputils.h"
 #include "qcompress.h"
 #include "value-editor/largetextmodel.h"
@@ -23,7 +25,8 @@ bool QmlUtils::isBinaryString(const QVariant &value) {
   if (!value.canConvert(QVariant::ByteArray)) {
     return false;
   }
-  QByteArray val = value.toByteArray();
+  QByteArray val = value.toByteArray();   
+
   return isBinary(val);
 }
 
@@ -44,6 +47,132 @@ QVariant QmlUtils::b64toByteArray(const QVariant &value)
     return QVariant(QByteArray::fromBase64(value.toString().toUtf8()));
 }
 
+QByteArray QmlUtils::minifyJSON(const QVariant &value)
+{
+    if (!value.canConvert(QVariant::ByteArray)) {
+      return QByteArray();
+    }
+
+    QByteArray val = value.toByteArray();
+
+    QByteArray minified;
+    minified.resize(val.size());
+
+    size_t new_length{};
+    auto error = simdjson::minify(val.data(), val.size(), minified.data(), new_length);
+
+    if (error != 0) {
+        qDebug() << "Failed to minify JSON with simdjson:" << error;
+        return QByteArray();
+    }
+
+    minified.resize(new_length);
+
+    return minified;
+}
+
+QByteArray QmlUtils::prettyPrintJSON(const QVariant &value)
+{
+    if (!value.canConvert(QVariant::ByteArray)) {
+      return QByteArray();
+    }
+
+    QByteArray val = value.toByteArray();
+    QByteArray result;
+    result.reserve(val.size() * 32);
+
+    const QByteArray whitespace("  ");
+    long level = 0;
+    bool ignore_next = false;
+    bool in_string = false;
+
+    // Based on https://github.com/alula/json-beautifier/blob/master/src/beautify.cpp
+    for (auto c : qAsConst(val)) {
+      switch (c) {
+        case '[':
+        case '{':
+          if (in_string) {
+            result.append(c);
+            break;
+          }
+          level++;
+          result.append(c);
+          result.append("\n");
+          for (long i = 0; i < level; i++) result.append(whitespace);
+          break;
+        case ']':
+        case '}':
+          if (in_string) {
+            result.append(c);
+            break;
+          }
+          if (level != 0) level--;
+          result.append("\n");
+          for (long i = 0; i < level; i++) result.append(whitespace);
+          result.append(c);
+          break;
+        case ',':
+          if (in_string) {
+            result.append(',');
+            break;
+          }
+          result.append(',');
+          result.append("\n");
+          for (long i = 0; i < level; i++) result.append(whitespace);
+          break;
+        case '\\':
+          if (ignore_next)
+            ignore_next = false;
+          else
+            ignore_next = true;
+          result.append("\\");
+          break;
+        case '"':
+          if (!ignore_next) in_string = !in_string;
+          result.append("\"");
+          break;
+        case ' ':
+          if (in_string) result.append(" ");
+          break;
+        case ':':
+          result.append(":");
+          if (!in_string) result.append(" ");
+          break;
+        case '\r':
+        case '\n':
+          break;
+        default:
+          if (ignore_next) ignore_next = false;
+          result.append(c);
+          break;
+      }
+    }
+
+    return result;
+}
+
+bool QmlUtils::isJSON(const QVariant &value)
+{
+    if (!value.canConvert(QVariant::ByteArray)) {
+      return false;
+    }
+
+    QByteArray val = value.toByteArray();
+    int originalSize = val.size();
+    val.resize(val.size() + simdjson::SIMDJSON_PADDING);
+
+    simdjson::dom::parser parser;
+    simdjson::dom::element data;
+    auto error = parser.parse(val.data(), originalSize, false).get(data);
+
+    if (error != simdjson::SUCCESS && error != simdjson::NUMBER_ERROR) {
+        qDebug() << "JSON is not valid:" << simdjson::error_message(error);
+        return false;
+    }
+
+    return true;
+}
+
 QVariant QmlUtils::decompress(const QVariant &value) {
   if (!value.canConvert(QVariant::ByteArray)) {
     return 0;

src/app/qmlutils.cpp

@@ -9,10 +9,13 @@
 class QmlUtils : public QObject
 {
     Q_OBJECT
-public:    
+public:
     Q_INVOKABLE bool isBinaryString(const QVariant &value);
     Q_INVOKABLE long binaryStringLength(const QVariant &value);    
     Q_INVOKABLE QVariant b64toByteArray(const QVariant &value);
+    Q_INVOKABLE QByteArray minifyJSON(const QVariant &value);
+    Q_INVOKABLE QByteArray prettyPrintJSON(const QVariant &value);
+    Q_INVOKABLE bool isJSON(const QVariant &value);
 
     Q_INVOKABLE unsigned isCompressed(const QVariant &value);
     Q_INVOKABLE QVariant decompress(const QVariant &value);

src/app/qmlutils.h

@@ -197,7 +197,7 @@ Dialog {
                             Layout.preferredHeight: 30
 
                             min: 1
-                            max: 2000000
+                            max: 20000000
                             value: 150000
                             label: qsTranslate("RDM","Maximum Formatted Value Size")
                             description: qsTranslate("RDM", "Size in bytes")

src/qml/GlobalSettings.qml

@@ -72,6 +72,11 @@ ApplicationWindow {
         property int valueSizeLimit: 150000
     }
 
+    Settings {
+        id: defaultFormatterSettings
+        category: "formatter_overrides"
+    }
+
     SystemPalette {
         id: sysPalette
     }

src/qml/app.qml

@@ -15,7 +15,15 @@ ComboBox {
     palette.dark: sysPalette.dark
     palette.window: sysPalette.window
 
-    function selectItem(txt) {
+    function selectItem(txt) {        
+        var res = _select(txt);
+
+        if (res >= 0) {
+            activated(res)
+        }
+    }
+
+    function _select(txt) {
         var index = find(txt)
 
         console.log("Index:", index)
@@ -24,6 +32,6 @@ ComboBox {
             currentIndex = index;
         }
 
-        activated(index)
+        return index
     }
 }