#5275 Add option to XMLWriter to prevent char 0 disabling output escaping

stroomdev66 · stroomdev66 · commit 2d17f3150395 · 2026-03-10T15:44:06.000Z
diff --git a/stroom-app/src/test/java/stroom/pipeline/TestXmlWriter.java b/stroom-app/src/test/java/stroom/pipeline/TestXmlWriter.java
@@ -50,7 +50,7 @@ void testUnicodeHandling() throws Exception {
         attributes.addAttribute("", "att", "att", "string", text);
 
         final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        final TransformerHandler th = XMLUtil.createTransformerHandler(new FatalErrorListener(), true);
+        final TransformerHandler th = XMLUtil.createTransformerHandler(new FatalErrorListener(), true, true);
         th.setResult(new StreamResult(baos));
 
         th.startDocument();
diff --git a/stroom-importexport/stroom-importexport-impl/src/main/java/stroom/importexport/impl/Config.java b/stroom-importexport/stroom-importexport-impl/src/main/java/stroom/importexport/impl/Config.java
@@ -72,7 +72,7 @@ void handleAttribute(final String property, final Object value) {
 
     public void write(final Writer writer, final String entityType) throws IOException {
         try {
-            final SAXTransformerFactory stf = (SAXTransformerFactory) TransformerFactoryFactory.newInstance();
+            final SAXTransformerFactory stf = (SAXTransformerFactory) TransformerFactoryFactory.newInstance(true);
             final TransformerHandler th = stf.newTransformerHandler();
             final Transformer transformer = th.getTransformer();
 
diff --git a/stroom-kafka/stroom-kafka-impl/src/main/java/stroom/kafka/pipeline/StandardKafkaProducer.java b/stroom-kafka/stroom-kafka-impl/src/main/java/stroom/kafka/pipeline/StandardKafkaProducer.java
@@ -232,7 +232,7 @@ public void fatalError(final TransformerException exception) {
                 };
 
                 try {
-                    this.xmlValueHandler = XMLUtil.createTransformerHandler(errorListener, false);
+                    this.xmlValueHandler = XMLUtil.createTransformerHandler(errorListener, false, true);
                     outputStream = new ByteArrayOutputStream();
                     xmlValueHandler.setResult(new StreamResult(outputStream));
                     xmlValueHandler.startDocument();
diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/filter/AbstractSamplingFilter.java b/stroom-pipeline/src/main/java/stroom/pipeline/filter/AbstractSamplingFilter.java
@@ -68,7 +68,7 @@ public void startProcessing() {
         errorListener = new ErrorListenerAdaptor(getElementId(), locationFactory, errorReceiverProxy);
 
         try {
-            this.handler = XMLUtil.createTransformerHandler(errorListener, false);
+            this.handler = XMLUtil.createTransformerHandler(errorListener, false, true);
 
         } catch (final TransformerConfigurationException e) {
             errorReceiverProxy.log(Severity.FATAL_ERROR,
diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/record/XMLRecordEmitter.java b/stroom-pipeline/src/main/java/stroom/pipeline/record/XMLRecordEmitter.java
@@ -56,7 +56,7 @@
 public class XMLRecordEmitter extends XMLFilterAdaptor implements HasElementId {
 
     private final SAXTransformerFactory transformerFactory = (SAXTransformerFactory) TransformerFactoryFactory
-            .newInstance();
+            .newInstance(true);
     private final MyWriter outputStreamWriter = new MyWriter(1000);
     private final List<DestinationProvider> appenders;
     private final List<StartPrefixMapping> prefixList = new ArrayList<>();
diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/writer/XMLWriter.java b/stroom-pipeline/src/main/java/stroom/pipeline/writer/XMLWriter.java
@@ -103,6 +103,7 @@ public class XMLWriter extends AbstractWriter implements XMLFilter {
     private String rootElement;
 
     private boolean indentOutput = false;
+    private boolean preventEscapeSwitching = false;
 
     //XSL related props in order to support <xsl:output>
     private final XsltStore xsltStore;
@@ -186,7 +187,8 @@ public void startProcessing() {
 
             final ErrorListener errorListener = new ErrorListenerAdaptor(getElementId(), locationFactory,
                     getErrorReceiver());
-            final TransformerHandler th = XMLUtil.createTransformerHandler(errorListener, indentOutput);
+            final TransformerHandler th = XMLUtil
+                    .createTransformerHandler(errorListener, indentOutput, preventEscapeSwitching);
 
             if (outputProperties != null) {
                 th.getTransformer().setOutputProperties(outputProperties);
@@ -569,6 +571,15 @@ public void setSuppressXSLTNotFoundWarnings(final boolean suppressXSLTNotFoundWa
         this.suppressXSLTNotFoundWarnings = suppressXSLTNotFoundWarnings;
     }
 
+    @PipelineProperty(
+            description = "Some inputs might contain control character 0 which turns off output escaping. " +
+                          "Set this to true to prevent that behaviour.",
+            defaultValue = "false",
+            displayPriority = 4)
+    public void setPreventEscapeSwitching(final boolean preventEscapeSwitching) {
+        this.preventEscapeSwitching = preventEscapeSwitching;
+    }
+
     /**
      * Used when XSLT is supplied (for xsl:output)
      */
diff --git a/stroom-pipeline/src/test/java/stroom/pipeline/xml/converter/json/TestJSONParser.java b/stroom-pipeline/src/test/java/stroom/pipeline/xml/converter/json/TestJSONParser.java
@@ -80,6 +80,11 @@ void testMulti() throws IOException {
         positiveTest("Multi");
     }
 
+    @Test
+    void testEmbeddedXml() throws IOException {
+        positiveTest("EmbeddedXml");
+    }
+
     private void negativeTest(final String stem, final String type) throws IOException {
         test(stem, "~" + type, true);
     }
@@ -299,6 +304,7 @@ private void test(final String stem, final String testType, final boolean expect
 
         final XMLWriter xmlWriter = new XMLWriter(errorReceiverProxy, null, null, null, null, null, null, null);
         xmlWriter.setIndentOutput(true);
+        xmlWriter.setPreventEscapeSwitching(true);
         xmlWriter.setTarget(xmlAppender);
 
         final JSONWriter jsonWriter = new JSONWriter(errorReceiverProxy);
diff --git a/stroom-pipeline/src/test/resources/TestJSON/EmbeddedXml.in b/stroom-pipeline/src/test/resources/TestJSON/EmbeddedXml.in
@@ -0,0 +1,12 @@
+{
+   "data" :
+   {
+      "structure" :
+      {
+         "values" :
+         {
+            "content" : "<XXXX>_aa_\u0000_bb_</XXXX>"
+         }
+      }
+   }
+}
diff --git a/stroom-pipeline/src/test/resources/TestJSON/EmbeddedXml.json_out b/stroom-pipeline/src/test/resources/TestJSON/EmbeddedXml.json_out
@@ -0,0 +1,9 @@
+{
+  "data" : {
+    "structure" : {
+      "values" : {
+        "content" : "<XXXX>_aa_\u0000_bb_</XXXX>"
+      }
+    }
+  }
+}
diff --git a/stroom-pipeline/src/test/resources/TestJSON/EmbeddedXml.xml_out b/stroom-pipeline/src/test/resources/TestJSON/EmbeddedXml.xml_out
@@ -0,0 +1,10 @@
+<?xml version="1.1" encoding="UTF-8"?>
+<map xmlns="http://www.w3.org/2013/XSL/json">
+   <map key="data">
+      <map key="structure">
+         <map key="values">
+            <string key="content">&lt;XXXX&gt;_aa_&#x0;_bb_&lt;/XXXX&gt;</string>
+         </map>
+      </map>
+   </map>
+</map>
diff --git a/stroom-util/src/main/java/stroom/util/xml/MyXmlEmitter.java b/stroom-util/src/main/java/stroom/util/xml/MyXmlEmitter.java
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // Copyright (c) 2015 Saxonica Limited.
 // This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 // If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 // This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0.
-////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+/// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 package stroom.util.xml;
 
@@ -73,6 +73,15 @@ public class MyXmlEmitter extends XMLEmitter {
         IS_SPECIAL_IN_ATT['\"'] = true;
     }
 
+    /**
+     * Use this flag to determine if we are going to let the control character 0 toggle escape switching.
+     */
+    private final boolean preventEscapeSwitching;
+
+    public MyXmlEmitter(final boolean preventEscapeSwitching) {
+        this.preventEscapeSwitching = preventEscapeSwitching;
+    }
+
     /**
      * Write contents of array to current writer, after escaping special characters.
      * This method converts the XML special characters (such as &lt; and &amp;) into their
@@ -139,7 +148,7 @@ protected void writeEscape(final CharSequence chars, final boolean inAttribute)
 
             // examine the special character that interrupted the scan
             final char c = chars.charAt(i);
-            if (c == 0) {
+            if (c == 0 && !preventEscapeSwitching) {
                 // used to switch escaping on and off
                 disabled = !disabled;
             } else if (disabled) {
diff --git a/stroom-util/src/main/java/stroom/util/xml/TransformerFactoryFactory.java b/stroom-util/src/main/java/stroom/util/xml/TransformerFactoryFactory.java
@@ -50,7 +50,7 @@ public final class TransformerFactoryFactory {
                 LOGGER.info(SYSPROP_SET_TO + factoryName + END);
             }
 
-            final TransformerFactory factory = newInstance();
+            final TransformerFactory factory = newInstance(false);
             LOGGER.info(IMP_USED + factory.getClass().getName());
         } catch (final Exception ex) {
             LOGGER.error("Unable to create new TransformerFactory!", ex);
@@ -61,13 +61,13 @@ private TransformerFactoryFactory() {
         // Utility class.
     }
 
-    public static TransformerFactory newInstance() {
+    public static TransformerFactory newInstance(final boolean preventEscapeSwitching) {
         final TransformerFactory factory = TransformerFactory.newInstance();
         secureProcessing(factory);
 
         final SaxonTransformerFactory saxonTransformerFactory = (SaxonTransformerFactory) factory;
         final Configuration configuration = saxonTransformerFactory.getConfiguration();
-        configuration.setSerializerFactory(new MySerializerFactory(configuration));
+        configuration.setSerializerFactory(new MySerializerFactory(configuration, preventEscapeSwitching));
 
         return factory;
     }
@@ -82,13 +82,17 @@ private static void secureProcessing(final TransformerFactory factory) {
 
     private static class MySerializerFactory extends SerializerFactory {
 
-        public MySerializerFactory(final Configuration config) {
+        private final boolean preventEscapeSwitching;
+
+        public MySerializerFactory(final Configuration config,
+                                   final boolean preventEscapeSwitching) {
             super(config);
+            this.preventEscapeSwitching = preventEscapeSwitching;
         }
 
         @Override
         protected Emitter newXMLEmitter(final Properties properties) {
-            return new MyXmlEmitter();
+            return new MyXmlEmitter(preventEscapeSwitching);
         }
     }
 }
diff --git a/stroom-util/src/main/java/stroom/util/xml/XMLUtil.java b/stroom-util/src/main/java/stroom/util/xml/XMLUtil.java
@@ -106,15 +106,15 @@ public static void prettyPrintXML(final XMLReader xmlReader,
                                       final InputSource inputSource,
                                       final Writer writer) {
         try {
-            final TransformerHandler handler = createTransformerHandler(new FatalErrorListener(), true);
+            final TransformerHandler handler = createTransformerHandler(new FatalErrorListener(), true, true);
             handler.setResult(new StreamResult(writer));
             xmlReader.setErrorHandler(new FatalErrorHandler());
             xmlReader.setContentHandler(handler);
             xmlReader.parse(inputSource);
 
         } catch (final TransformerConfigurationException
-                | IOException
-                | SAXException e) {
+                       | IOException
+                       | SAXException e) {
             throw new RuntimeException(e);
         }
     }
@@ -133,13 +133,15 @@ public static void prettyPrintXML(final Reader reader, final Writer writer) {
 
     public static TransformerHandler createTransformerHandler(final boolean indentOutput)
             throws TransformerConfigurationException {
-        return createTransformerHandler(null, indentOutput);
+        return createTransformerHandler(null, indentOutput, true);
     }
 
     public static TransformerHandler createTransformerHandler(final ErrorListener errorListener,
-                                                              final boolean indentOutput)
+                                                              final boolean indentOutput,
+                                                              final boolean preventEscapeSwitching)
             throws TransformerConfigurationException {
-        final SAXTransformerFactory stf = (SAXTransformerFactory) TransformerFactoryFactory.newInstance();
+        final SAXTransformerFactory stf = (SAXTransformerFactory) TransformerFactoryFactory
+                .newInstance(preventEscapeSwitching);
         if (errorListener != null) {
             stf.setErrorListener(errorListener);
         }
diff --git a/unreleased_changes/20260310_154347_293__5275.md b/unreleased_changes/20260310_154347_293__5275.md
@@ -0,0 +1,68 @@
+* Bug **#5275** : Add option to XMLWriter to prevent char 0 disabling output escaping.
+
+
+```sh
+# ********************************************************************************
+# Issue number: 5275
+# Issue title:  jsonParser not escaping content correctly
+# Issue tags:   f:pipeline-processing, p:high
+# Issue link:   https://github.com/gchq/stroom/issues/5275
+# ********************************************************************************
+
+# ONLY the top line will be included as a change entry in the CHANGELOG.
+# The entry should be in GitHub flavour markdown and should be written on a SINGLE
+# line with no hard breaks. You can have multiple change files for a single GitHub issue.
+# The  entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
+# 'Fixed nasty bug'.
+#
+# Examples of acceptable entries are:
+#
+#
+# * Bug **#123** : Fix bug with an associated GitHub issue in this repository.
+#
+# * Bug **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository.
+#
+# * Feature **#789** : Add new feature X.
+#
+# * Bug : Fix bug with no associated GitHub issue.
+#
+#
+# Note: The line must start '* XXX ', where 'XXX' is a valid category,
+#       one of [Bug Feature Refactor Dependency Build].
+
+
+# --------------------------------------------------------------------------------
+# The following is random text to make this file unique for git's change detection
+# TYh1ldOKXx1PEui9OtuV0dXs1pflzTyqRAxoKM5Kbif3rxeo8QGqiotUuWX1v92wkBJGIF1NLVcWSZcW
+# S226ZZqL0CF8E4uL3UlcLuu7bhv3cEspAy388dKdlYySNGf9pyrkca7wD0PsHOmW8HHSaDteTYLworVJ
+# ZbvpZQaBAeJaQKUaXhJE7r8rex4ZNozhOD568ucRF7JGZRfENEL4XDucrjmM1EmydJ3yRTg3KulVsqMF
+# d2eMU1fOiuj8nkRc5zfkJ91Ejj33GVOTqTQUm1x1GzIW8CR6ZVgNJzWpShoqNJYGY1XQShqlXQi92o3b
+# UJfbLBx7GXAKApqFQolbGdkU60cxgr4pC62C7m9DTavwtaxvWJYoCeiCP8BkrlDnRuZuNe37UqgfF8q2
+# qjSbRPDf8bRqSdZYpzTS3S3AQzLESLjRUk9ZfyJvAJx6KgLAReafhhkfmDUdhUDa6mUd5kXliDklmN8t
+# r4y4AmJkrfWTcUfVVJzknCtbhCTWrY81DYhQZoOw6RpCgrGR5Yuj5ElRta73CizKM01az2yjKSWUCvBP
+# 4AoMipOLH9iRQ959Q4DylkoWidHZcKcxmpy33iYW0AwWyH8t9wcgsYqCr984fKDuwhcncxCLQyhRZRGw
+# 3yX8yfkQnHJ1VoW2pGhnuPHE8IU3XDubbAhh5RnLGlTmGJCa097a6hUGxZemHJmnF5CZW8KtQSpNmPHp
+# 2IskmtpfcOhOJ72DgzcsB5KYTmrIO3xStH1vKqtiUnu0jaKqg6ZRflPafwa8YxqSfg0DuMYXDbM8dZyT
+# TsVgp9P0vtb8unJowYnFkyBceUVdgjhqXFfkRyR8754j0Nd59CUaewyRvuJedWe9BdBxJbxC6GB6j6U6
+# MPc4Do8DGRGH2nvkpnfCKGTGGruKB3WtK76L3AZdnVe2lTyFyo1VbK1vBzVClcLF3Pj6Don1zKa4hZNo
+# H4iH9FGOeARcimbLxuDFeNy5YZMRf8aTR5Ig3N9PGxjIVrlclgHywiBqIK5l4d3UfBodCLM1xuVcOtXG
+# Sc8c08TX8iOh4iPt7tkZfZGkdZgftME2D8O9y3kBIlhFZETe2UEOPUCI64lH19EgMoMk5fpVv5FldPO9
+# AIfTrT9fjLDbCVqqGuxXhyFtN08h1wutSsvsxyXfueP4PTaEkK5oAQBnHGaF0B8mr86fuQSBwlEk54NY
+# ytUFsGUq99q3DfY6r7Lqqo0TleoEI4OmD6DUt414ucYNNURvh0t0SC3BhSGvT68fEvIo1Wy2WtniwzKn
+# BcWDxzuxg31c8i4CO77Vubds3sbGna5Fyg14O0YDRF7h6SQRaNl01aLeFjBsta3nEvrOYqDVhS6GwFZR
+# GFI8dLeGFFUUZg7z21FRuPJMvm71wLUHbJv1LrNBMx95WDOeDuQipbLBJ52NiIHPmGRQdcFdkMr7hYqX
+# 4z4C1i5eZ8Gog8YVmgib1UCcGFFsVzfznlXKMk3heyGUd5Bc8h3uCndDIlL3CPlmNpCrL2JQMHKRGaE7
+# 3nb8Wy0kkeirrRhT57YCxCIAV5dshXxgPhlmjp4KmDgOobRSD5NUcisHah8WyZcyw6d7TTlzFVjwprKz
+# UTthpCm0aJCpOpNinnywofKvEbUDKjUhTOZMbuRokoN3HjoZU84lQhfkbMARgiTTKH3RCwm80fAiXqsi
+# DSYTGdpwHu9dMcX2FkWAhondieZswqrgIjOApu6WJnmt7GLVrD12u5gOGmJMwHCtKSIZIHoLoKPlpIQI
+# Rnk2s1IP8ySnQ7OL0HpNrOCigkgarpi3vWbEdidinCgs1NJOSMucoOJRqSXYJX3COcYUD8hwyFaIK7LS
+# UCuXrcS4GeJWVZLhOJTS2sTtgnWoOIoImg5iGkmfmO0ahtctWkWfMwsJEtIZ0wd7NiyoXXaUc1fyhBqk
+# QxrhCzKpSHCqxYucUSDl8x8unOwhzV32zjY4NDkW1FxTyQHnfkNKDbJ6eraRjBi0sZvnytkuwrtVkSMS
+# N7kq2aGym3f24x4lSEj9rMuhqt26PGJz5BkIKRlbxPStfJ8UvjMBRTWMZSOnliQUc5f6gDzRfy1bx6Yc
+# 5O70Fe8Lv8uMD3p6E6PRQenkGTfBXevplPLnvLMAvAVuQOU3PeXlJ24Oq4mgX2d25V6mXLm2uErMAGP7
+# 04k59nDFxfhhptIaoDMSOKUKrUXK3eQzIOj3vFEjmDcmFFdnVh69t6DwFim4JHweqCmkYjKjfjqIQ3Gr
+# FK4q3PEwiqAfZNIO7pLYbYYBwv15OB6J379YBBEAoijQ9dQGdY4KztErd74pU4JaO4grPq8WCQHrqkpz
+# sdMAPVxcvf2r2VsMqWueLzigtturspgB1FEmTBzuIi5ZfImKrkKt9tNJV20mzvBAUnQcu3QbH0JYOwRD
+# --------------------------------------------------------------------------------
+
+```
