Merge branch '2.11'

Author: cowtowncoder

csv/src/main/java/com/fasterxml/jackson/dataformat/csv/impl/CsvDecoder.java

@@ -25,7 +25,6 @@ public class CsvDecoder {
 
     private final static int INT_CR = '\r';
     private final static int INT_LF = '\n';
-//    private final static int INT_HASH = '#';
 
     /*
     /**********************************************************************
@@ -488,43 +487,68 @@ public boolean startNewLine() throws IOException {
      * @since 2.10
      */
     public boolean skipLinesWhenNeeded() throws IOException {
-        if (!(_allowComments || _skipBlankLines)) {
+        if (_allowComments) {
+            return _skipCommentLines();
+        }
+        if (!_skipBlankLines) {
             return hasMoreInput();
         }
-        int firstCharacterPtr = _inputPtr;
+
+        // only need to skip fully empty lines
         while (hasMoreInput()) {
-            char ch = _inputBuffer[_inputPtr++];
+            char ch = _inputBuffer[_inputPtr];
             if (ch == '\r' || ch == '\n') {
+                ++_inputPtr;
                 _pendingLF = ch;
                 _handleLF();
-                // track the start of the new line
-                firstCharacterPtr = _inputPtr;
                 continue;
             }
-            if (ch == ' ') {
+            if (ch != ' ') {
+                return true; // processing can go on
+            }
+            ++_inputPtr;
+        }
+        return false; // end of input
+    }
+
+    public boolean _skipCommentLines() throws IOException
+    {
+        while ((_inputPtr < _inputEnd) || loadMore()) {
+            char ch = _inputBuffer[_inputPtr];
+            switch (ch) {
+            case '#':
+                ++_inputPtr;
+                _skipCommentContents();
+                continue;
+            case '\r':
+            case '\n':
+                ++_inputPtr;
+                _pendingLF = ch;
+                _handleLF();
+                continue;
+            case ' ':
                 // skip all blanks (in both comments/blanks skip mode)
+                ++_inputPtr;
                 continue;
+            default:
+                return true;
             }
-            if (_allowComments) {
-                if (_inputBuffer[firstCharacterPtr] == '#') {
-                    // on a commented line, skip everything
-                    continue;
-                }
-                if (ch == '#') {
-                    // we reach this point when whitespaces precedes the hash character
-                    // move the firstCharacterPtr to the '#' location in order to skip the line completely
-                    firstCharacterPtr = _inputPtr-1;
-                    continue;
-                }
-            }
-            // we reached a non skippable character, this line needs to be parsed
-            // rollback the input pointer to the beginning of the line
-            _inputPtr = firstCharacterPtr;
-            return true; // processing can go on
         }
         return false; // end of input
     }
 
+    private void _skipCommentContents() throws IOException
+    {
+        while ((_inputPtr < _inputEnd) || loadMore()) {
+            char ch = _inputBuffer[_inputPtr++];
+            if (ch == '\r' || ch == '\n') {
+                _pendingLF = ch;
+                _handleLF();
+                break;
+            }
+        }
+    }
+
     /**
      * Method called to blindly skip a single line of content, without considering
      * aspects like quoting or escaping. Used currently simply to skip the first

csv/src/test/java/com/fasterxml/jackson/dataformat/csv/deser/SkipBlankLines15Test.java renamed to csv/src/test/java/com/fasterxml/jackson/dataformat/csv/deser/SkipEmptyLines15Test.java

@@ -9,7 +9,7 @@
 import static org.junit.Assert.assertArrayEquals;
 
 // for [dataformats-text#15]: Allow skipping of empty lines
-public class SkipBlankLines15Test extends ModuleTestBase {
+public class SkipEmptyLines15Test extends ModuleTestBase {
 
     private static final String CSV_WITH_EMPTY_LINE = "1,\"xyz\"\n\ntrue,\n";
     private static final String CSV_WITH_BLANK_LINE = "1,\"xyz\"\n   \ntrue,\n";
@@ -129,14 +129,18 @@ public void testCsvWithBlankLineAndCommentSkipBlankLinesFeatureDisabled() throws
         ), rows);
     }
 
+    // 14-Apr-2020, tatu: Due to [dataformats-text#191], can not retain leading spaces
+    //   when trimming empty lines and/or comments, so test changed for 2.11
     public void testCsvWithBlankLineAndCommentSkipBlankLinesFeatureEnabled() throws Exception {
         String[][] rows = mapperForCsvAsArray()
                 .with(CsvParser.Feature.SKIP_EMPTY_LINES)
                 .readValue(CSV_WITH_BLANK_LINE_AND_COMMENT);
         // blank/empty lines are skipped
         assertArrayEquals(expected(
                 row("1", "xyz"),
-                row("  #comment"),
+                // As per: [dataformats-text#191]
+//                row("  #comment"),
+                row("#comment"),
                 row("true", "")
         ), rows);
     }

csv/src/test/java/com/fasterxml/jackson/dataformat/csv/failing/ParserSkipEmpty191Test.java renamed to csv/src/test/java/com/fasterxml/jackson/dataformat/csv/deser/SkipEmptyLines191Test.java

@@ -1,6 +1,7 @@
-package com.fasterxml.jackson.dataformat.csv.failing;
+package com.fasterxml.jackson.dataformat.csv.deser;
 
-import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
 import java.util.List;
 import java.util.Map;
 
@@ -9,25 +10,26 @@
 import com.fasterxml.jackson.dataformat.csv.ModuleTestBase;
 
 // [dataformats-text#191]
-public class ParserSkipEmpty191Test extends ModuleTestBase {
+public class SkipEmptyLines191Test extends ModuleTestBase {
 
     private static String COL_1 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
     private static String COL_2 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
 
     // [dataformats-text#191]: IndexArrayOutOfBounds at 4000
-    public void testBigCsvFile() throws Exception {
+    public void testBigCsvFile() throws Exception
+    {
         CsvSchema schema = CsvSchema
                 .emptySchema()
                 .withHeader()
                 .withColumnSeparator(';')
                 .withNullValue("null")
                 .withComments();
 
-        try (InputStream csvFileStream = getClass().getResourceAsStream("/csv/issue-191.csv")) {
+        try (Reader r = new StringReader(_generate4kDoc())) {
             List<Map<String, String>> result = new CsvMapper()
                     .readerFor(Map.class)
                     .with(schema)
-                    .<Map<String, String>>readValues(csvFileStream)
+                    .<Map<String, String>>readValues(r)
                     .readAll();
 
             for (Map<String, String> row : result) {
@@ -36,4 +38,16 @@ public void testBigCsvFile() throws Exception {
             }
         }
     }
-}
+
+    private String _generate4kDoc() {
+        StringBuilder sb = new StringBuilder(5000)
+.append("COL_1;COL_2\n")
+.append("# csv file with a newline at char 4000 (the buffer size) to verify a bug\n")
+.append("# LF has to be used for newlines to work\n")
+.append("# alignment chars to have the newline as the 4000s char: ----------------\n");
+        for (int i = 0; i < 40; ++i) {
+            sb.append("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa;bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n");
+        }
+        return sb.toString();
+    }
+}

csv/src/test/resources/csv/issue-191.csv

@@ -18,6 +18,8 @@ Modules:
 #180: (yaml) YAMLGenerator serializes string with special chars unquoted when
   using `MINIMIZE_QUOTES` mode
  (reported, fix contributed by Timo R)
+#191: (csv) `ArrayIndexOutOfBoundsException` when skipping empty lines, comments
+ (reported by f-julian@github)
 
 2.10.4 (not yet released)