shared-string: further reduce memory consumption of json-writer

... by allocating repeatedly used strings in a global set

Author: kdudka

CMakeLists.txt

@@ -38,6 +38,7 @@ add_library(cs STATIC
     instream.cc
     json-parser.cc
     json-writer.cc
+    shared-string.cc
     version.cc
 )
 

json-writer.cc

@@ -19,13 +19,17 @@
 
 #include "json-writer.hh"
 
+#include "shared-string-ptree.hh"
+
 #include <queue>
 
 #include <boost/foreach.hpp>
 #include <boost/iostreams/filtering_stream.hpp>
 #include <boost/iostreams/filter/regex.hpp>
 #include <boost/property_tree/json_parser.hpp>
 
+typedef SharedStringPTree PTree;
+
 struct JsonWriter::Private {
     std::ostream                   &str;
     std::queue<Defect>              defQueue;
@@ -54,13 +58,13 @@ void JsonWriter::setScanProps(const TScanProps &scanProps) {
     d->scanProps = scanProps;
 }
 
-void appendDefectNode(boost::property_tree::ptree &dst, const Defect &def) {
+void appendDefectNode(PTree &dst, const Defect &def) {
     using std::string;
 
     // go through events
-    boost::property_tree::ptree evtList;
+    PTree evtList;
     BOOST_FOREACH(const DefEvent &evt, def.events) {
-        boost::property_tree::ptree evtNode;
+        PTree evtNode;
 
         // describe the location
         evtNode.put<string>("file_name", evt.fileName);
@@ -78,7 +82,7 @@ void appendDefectNode(boost::property_tree::ptree &dst, const Defect &def) {
     }
 
     // create a node for a single defect
-    boost::property_tree::ptree defNode;
+    PTree defNode;
     defNode.put<string>("checker", def.checker);
     if (!def.annotation.empty())
         defNode.put<string>("annotation", def.annotation);
@@ -123,18 +127,18 @@ void JsonWriter::flush() {
     str.push(d->str);
 
     // encode scan properties if we have some
-    boost::property_tree::ptree root;
+    PTree root;
     if (!d->scanProps.empty()) {
-        boost::property_tree::ptree scan;
+        PTree scan;
         BOOST_FOREACH(TScanProps::const_reference prop, d->scanProps)
             scan.put<std::string>(prop.first, prop.second);
 
         root.put_child("scan", scan);
     }
 
     // node representing the list of defects
-    root.put_child("defects", boost::property_tree::ptree());
-    boost::property_tree::ptree &defects = root.get_child("defects");
+    root.put_child("defects", PTree());
+    PTree &defects = root.get_child("defects");
 
     // go through the queue and move defects one by one to the property tree
     for (; !d->defQueue.empty(); d->defQueue.pop())

shared-string-ptree.hh

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This file is part of csdiff.
+ *
+ * csdiff is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ *
+ * csdiff is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with csdiff.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef H_GUARD_SHARED_STRING_PTREE_H
+#define H_GUARD_SHARED_STRING_PTREE_H
+
+#include "shared-string.hh"
+
+#include <boost/property_tree/ptree.hpp>
+
+template <typename T>
+struct SharedStrTrans {
+    typedef SharedStr internal_type;
+    typedef SharedStr external_type;
+
+    boost::optional<SharedStr> put_value(const T &val) const {
+        const SharedStr ss(val);
+        return ss;
+    }
+
+    boost::optional<T> get_value(const SharedStr &ss) const {
+        T val;
+        ss.writeOut(&val);
+        return val;
+    }
+};
+
+namespace boost {
+    namespace property_tree {
+        template <typename T>
+        struct translator_between<SharedStr, T> {
+            typedef SharedStrTrans<T> type;
+        };
+    }
+}
+
+typedef boost::property_tree::basic_ptree<std::string, SharedStr>
+    SharedStringPTree;
+
+#endif /* H_GUARD_SHARED_STRING_PTREE_H */

shared-string.cc

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This file is part of csdiff.
+ *
+ * csdiff is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ *
+ * csdiff is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with csdiff.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "shared-string.hh"
+
+#include <sstream>
+
+SharedStr::TStor SharedStr::stor_;
+
+SharedStr::SharedStr():
+    iter_(stor_.end())
+{
+}
+
+void SharedStr::swap(SharedStr &ref)
+{
+    std::swap(iter_, ref.iter_);
+}
+
+void SharedStr::hashStr(const std::string &str)
+{
+    iter_ = stor_.insert(str).first;
+}
+
+SharedStr::SharedStr(const int i)
+{
+    std::ostringstream ss;
+    ss << i;
+    this->hashStr(ss.str());
+}
+
+SharedStr::SharedStr(const std::string &str)
+{
+    this->hashStr(str);
+}
+
+void SharedStr::writeOut(std::string *pDst) const {
+    if (stor_.end() == iter_)
+        pDst->clear();
+    else
+        *pDst = *iter_;
+}

shared-string.hh

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This file is part of csdiff.
+ *
+ * csdiff is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ *
+ * csdiff is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with csdiff.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef H_GUARD_SHARED_STRING_H
+#define H_GUARD_SHARED_STRING_H
+
+#include <set>
+#include <string>
+
+class SharedStr {
+    public:
+        SharedStr();
+        SharedStr(int i);
+        SharedStr(const std::string &str);
+        void writeOut(std::string *pDst) const;
+        void swap(SharedStr &);
+
+    private:
+        typedef std::set<std::string>   TStor;
+        static TStor                    stor_;
+        TStor::iterator                 iter_;
+        void hashStr(const std::string &);
+};
+
+#endif /* H_GUARD_SHARED_STRING_H */