Merge pull request #651 from the-thing/650/truncated_filestore_messages

`FileStore` message length fix

(cherry picked from commit e050d3d)

quickfixj-core/src/main/java/org/quickfixj/CharsetSupport.java

@@ -64,6 +64,10 @@ public static void setCharset(String charset) throws UnsupportedEncodingExceptio
         CharsetSupport.isStringEquivalent = isStringEquivalent(charsetInstance);
     }
 
+    public static void setDefaultCharset() throws UnsupportedEncodingException {
+        setCharset(getDefaultCharset());
+    }
+
     public static String getCharset() {
         return charset;
     }

quickfixj-core/src/main/java/quickfix/CachedFileStore.java

@@ -28,6 +28,7 @@
 import java.io.BufferedOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -86,8 +87,6 @@ public class CachedFileStore implements MessageStore {
 
     private FileOutputStream headerFileOutputStream;
 
-    private final String charsetEncoding = CharsetSupport.getCharset();
-
     CachedFileStore(String path, SessionID sessionID, boolean syncWrites) throws IOException {
         this.syncWrites = syncWrites;
 
@@ -308,16 +307,15 @@ public boolean get(int sequence, String message) {
         throw new UnsupportedOperationException("not supported");
     }
 
-    private String read(long offset, long size) throws IOException {
-        final byte[] data = new byte[(int) size];
-
-        messageFileReader.seek(offset);
-        if (messageFileReader.read(data) != size) {
-            throw new IOException("Truncated input while reading message: "
-                    + new String(data, charsetEncoding));
+    private String read(long offset, int size) throws IOException {
+        try {
+            final byte[] data = new byte[size];
+            messageFileReader.seek(offset);
+            messageFileReader.readFully(data);
+            return new String(data, CharsetSupport.getCharset());
+        } catch (EOFException eofe) { // can't read fully
+            throw new IOException("Truncated input while reading message: offset=" + offset + ", expected size=" + size, eofe);
         }
-
-        return new String(data, charsetEncoding);
     }
 
     private Collection<String> getMessage(long startSequence, long endSequence) throws IOException {
@@ -326,7 +324,7 @@ private Collection<String> getMessage(long startSequence, long endSequence) thro
         final List<long[]> offsetAndSizes = messageIndex.get(startSequence, endSequence);
         for (final long[] offsetAndSize : offsetAndSizes) {
             if (offsetAndSize != null) {
-                final String message = read(offsetAndSize[0], offsetAndSize[1]);
+                final String message = read(offsetAndSize[0], (int) offsetAndSize[1]);
                 messages.add(message);
             }
         }
@@ -341,7 +339,8 @@ private Collection<String> getMessage(long startSequence, long endSequence) thro
      */
     public boolean set(int sequence, String message) throws IOException {
         final long offset = messageFileWriter.getFilePointer();
-        final int size = message.length();
+        final byte[] messageBytes = message.getBytes(CharsetSupport.getCharset());
+        final int size = messageBytes.length;
         messageIndex.put((long) sequence, new long[] { offset, size });
         headerDataOutputStream.writeInt(sequence);
         headerDataOutputStream.writeLong(offset);
@@ -350,7 +349,7 @@ public boolean set(int sequence, String message) throws IOException {
         if (syncWrites) {
             headerFileOutputStream.getFD().sync();
         }
-        messageFileWriter.write(message.getBytes(CharsetSupport.getCharset()));
+        messageFileWriter.write(messageBytes);
         return true;
     }
 

quickfixj-core/src/main/java/quickfix/FileStore.java

@@ -63,7 +63,6 @@ public class FileStore implements MessageStore, Closeable {
     private final String sessionFileName;
     private final boolean syncWrites;
     private final int maxCachedMsgs;
-    private final String charsetEncoding = CharsetSupport.getCharset();
     private RandomAccessFile messageFileReader;
     private RandomAccessFile messageFileWriter;
     private DataOutputStream headerDataOutputStream;
@@ -350,7 +349,7 @@ private String getMessage(long offset, int size, int i) throws IOException {
             final byte[] data = new byte[size];
             messageFileReader.seek(offset);
             messageFileReader.readFully(data);
-            return new String(data, charsetEncoding);
+            return new String(data, CharsetSupport.getCharset());
         } catch (EOFException eofe) { // can't read fully
             throw new IOException("Truncated input while reading message: messageIndex=" + i
                     + ", offset=" + offset + ", expected size=" + size, eofe);
@@ -363,7 +362,8 @@ private String getMessage(long offset, int size, int i) throws IOException {
     @Override
     public boolean set(int sequence, String message) throws IOException {
         final long offset = messageFileWriter.getFilePointer();
-        final int size = message.length();
+        final byte[] messageBytes = message.getBytes(CharsetSupport.getCharset());
+        final int size = messageBytes.length;
         if (messageIndex != null) {
             updateMessageIndex(sequence, offset, size);
         }
@@ -374,7 +374,7 @@ public boolean set(int sequence, String message) throws IOException {
         if (syncWrites) {
             headerFileOutputStream.getFD().sync();
         }
-        messageFileWriter.write(message.getBytes(CharsetSupport.getCharset()));
+        messageFileWriter.write(messageBytes);
         return true;
     }
 

quickfixj-core/src/test/java/quickfix/AbstractMessageStoreTest.java

@@ -20,11 +20,14 @@
 package quickfix;
 
 import junit.framework.TestCase;
+import org.quickfixj.CharsetSupport;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.List;
 
 public abstract class AbstractMessageStoreTest extends TestCase {
+
     private SessionID sessionID;
     private MessageStore store;
 
@@ -167,6 +170,44 @@ public void testRefreshMessageStore() throws Exception {
         }
     }
 
+    public void testSetAndGetMessageWithAsciiCharacters() throws IOException {
+        MessageStore underTest = getStore();
+
+        if (underTest instanceof SleepycatStore) {
+            return;
+        }
+
+        underTest.set(1, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVXYZ0123456789");
+
+        List<String> messages = new ArrayList<>();
+        underTest.get(1, 1, messages);
+
+        assertEquals(1, messages.size());
+        assertEquals("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVXYZ0123456789", messages.get(0));
+    }
+
+    public void testSetAndGetMessageWithUnicodeCharacters() throws IOException {
+        MessageStore underTest = getStore();
+
+        if (underTest instanceof SleepycatStore) {
+            return;
+        }
+
+        CharsetSupport.setCharset("UTF-8");
+
+        try {
+            underTest.set(1, "a \u00A9 \u2603 \uD834\uDF06");
+
+            List<String> messages = new ArrayList<>();
+            underTest.get(1, 1, messages);
+
+            assertEquals(1, messages.size());
+            assertEquals("a \u00A9 \u2603 \uD834\uDF06", messages.get(0));
+        } finally {
+            CharsetSupport.setDefaultCharset();
+        }
+    }
+
     protected void closeMessageStore(MessageStore store) throws IOException {
         // does nothing, by default
     }

quickfixj-core/src/test/java/quickfix/FieldTest.java

@@ -19,6 +19,14 @@
 
 package quickfix;
 
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.util.Arrays;
+import java.util.Date;
+
 import org.junit.Test;
 import org.quickfixj.CharsetSupport;
 import quickfix.field.ClOrdID;
@@ -32,14 +40,6 @@
 import quickfix.fix50.MarketDataIncrementalRefresh;
 import quickfix.fix50.NewOrderSingle;
 
-import java.io.UnsupportedEncodingException;
-import java.math.BigDecimal;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.util.Arrays;
-import java.util.Date;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -89,7 +89,7 @@ public void testFieldCalculationsWithUTF8Charset() throws UnsupportedEncodingExc
         try {
             testFieldCalcuations("\u6D4B\u9A8C\u6570\u636E", 50, 16);
         } finally {
-            CharsetSupport.setCharset(CharsetSupport.getDefaultCharset());
+            CharsetSupport.setDefaultCharset();
         }
     }
 

quickfixj-core/src/test/java/quickfix/FileStoreTest.java

@@ -19,15 +19,18 @@
 
 package quickfix;
 
+import org.quickfixj.CharsetSupport;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
 public class FileStoreTest extends AbstractMessageStoreTest {
 
-    protected void tearDown() throws Exception {
+    public void tearDown() throws Exception {
         super.tearDown();
+        CharsetSupport.setDefaultCharset();
         FileStore fileStore = (FileStore) getStore();
         try {
             fileStore.closeAndDeleteFiles();
@@ -36,6 +39,7 @@ protected void tearDown() throws Exception {
         }
     }
 
+    @Override
     protected MessageStoreFactory getMessageStoreFactory() throws ConfigError, FieldConvertError {
         SessionSettings settings = new SessionSettings(getConfigurationFileName());
         // Initialize the session settings from the defaults
@@ -44,6 +48,7 @@ protected MessageStoreFactory getMessageStoreFactory() throws ConfigError, Field
         return new FileStoreFactory(settings);
     }
 
+    @Override
     protected Class<?> getMessageStoreClass() {
         return FileStore.class;
     }

quickfixj-core/src/test/java/quickfix/MessageTest.java

@@ -316,7 +316,7 @@ private void doTestMessageWithEncodedField(String charset, String text) throws E
             final Message msg = new Message(order.toString(), DataDictionaryTest.getDictionary());
             assertEquals(charset + " encoded field", text, msg.getString(EncodedText.FIELD));
         } finally {
-            CharsetSupport.setCharset(CharsetSupport.getDefaultCharset());
+            CharsetSupport.setDefaultCharset();
         }
     }
 

quickfixj-core/src/test/java/quickfix/SleepycatStoreTest.java

@@ -23,7 +23,8 @@
 import java.io.IOException;
 
 public class SleepycatStoreTest extends AbstractMessageStoreTest {
-    protected MessageStoreFactory getMessageStoreFactory() throws ConfigError, FieldConvertError {
+
+    protected MessageStoreFactory getMessageStoreFactory() throws ConfigError {
         SessionSettings settings = new SessionSettings(getConfigurationFileName());
         File tmpfile;
         try {
@@ -62,5 +63,4 @@ public void testCloseAndOpen() throws Exception {
         assertEquals(123, store.getNextSenderMsgSeqNum());
         assertEquals(321, store.getNextTargetMsgSeqNum());
     }
-
 }

quickfixj-core/src/test/java/quickfix/mina/message/FIXMessageDecoderTest.java

@@ -55,24 +55,21 @@ public class FIXMessageDecoderTest {
 
     @Before
     public void setUp() throws Exception {
+        CharsetSupport.setDefaultCharset();
         decoder = new FIXMessageDecoder();
         buffer = IoBuffer.allocate(8192);
         decoderOutput = new ProtocolDecoderOutputForTest();
     }
 
     @After
     public void tearDown() throws Exception {
+        CharsetSupport.setDefaultCharset();
         buffer.clear();
     }
 
-    @Test
+    @Test(expected = UnsupportedEncodingException.class)
     public void testInvalidStringCharset() throws Exception {
-        try {
-            decoder = new FIXMessageDecoder("BOGUS");
-            fail("no exception thrown");
-        } catch (UnsupportedEncodingException e) {
-            // expected
-        }
+        decoder = new FIXMessageDecoder("BOGUS");
     }
 
     @Test
@@ -104,8 +101,6 @@ public void testWesternEuropeanDecoding() throws Exception {
             doWesternEuropeanDecodingTest();
         } catch (InvalidMessage e) {
             // expected
-        } finally {
-            CharsetSupport.setCharset(CharsetSupport.getDefaultCharset());
         }
     }