Support PostgreSQL tagged dollar strings and preserve literal bodies

README.md

@@ -148,6 +148,10 @@ Beyond statement shapes, the grammar handles nested sub-selects, bind parameters
 array-literal ambiguity. The complete reference is on the
 [syntax page](https://jsqlparser.github.io/JSqlParser/syntax.html).
 
+PostgreSQL dollar-quoted strings, including `$tag$…$tag$`, retain their delimiter and
+literal body in `StringValue`. For dialects that use the same spelling as an unquoted
+identifier, `parser.withDollarQuotedStringTags(false)` retains identifier parsing.
+
 ## Statement classification
 
 Any parsed statement can say what it actually does — no second parse, no visitor to write:

src/main/java/net/sf/jsqlparser/expression/StringValue.java

@@ -42,10 +42,14 @@ public StringValue(String escapedValue) {
             value = escapedValue.substring(1, escapedValue.length() - 1);
             quoteStr = "\"";
             return;
-        } else if (escapedValue.length() >= 4 && escapedValue.startsWith("$$")
-                && escapedValue.endsWith("$$")) {
-            value = escapedValue.substring(2, escapedValue.length() - 2);
-            quoteStr = "$$";
+        }
+
+        String delimiter = getDollarQuoteDelimiter(escapedValue);
+        if (delimiter != null && escapedValue.length() >= 2 * delimiter.length()
+                && escapedValue.endsWith(delimiter)) {
+            quoteStr = delimiter;
+            value = escapedValue.substring(delimiter.length(),
+                    escapedValue.length() - delimiter.length());
             return;
         }
 
@@ -64,6 +68,32 @@ public StringValue(String escapedValue) {
         value = escapedValue;
     }
 
+    /**
+     * Returns the opening PostgreSQL dollar-quote delimiter, or null if there is none. A tag
+     * follows unquoted identifier rules, excluding dollar signs. This method does not require the
+     * closing delimiter or inspect the body.
+     */
+    public static String getDollarQuoteDelimiter(String text) {
+        if (text == null || text.length() < 2 || text.charAt(0) != '$') {
+            return null;
+        }
+        int end = text.indexOf('$', 1);
+        if (end < 0) {
+            return null;
+        }
+        for (int i = 1; i < end;) {
+            int character = text.codePointAt(i);
+            boolean valid =
+                    i == 1 ? Character.isUnicodeIdentifierStart(character) || character == '_'
+                            : Character.isUnicodeIdentifierPart(character);
+            if (!valid) {
+                return null;
+            }
+            i += Character.charCount(character);
+        }
+        return text.substring(0, end + 1);
+    }
+
     public String getValue() {
         return value;
     }
@@ -90,6 +120,9 @@ public StringValue setQuoteStr(String quoteStr) {
     }
 
     public String getNotExcapedValue() {
+        if (quoteStr != null && quoteStr.startsWith("$")) {
+            return value;
+        }
         StringBuilder buffer = new StringBuilder(value);
         int index = 0;
         int deletesNum = 0;

src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java

@@ -143,6 +143,11 @@ public P withBackslashEscapeCharacter(boolean allowBackslashEscapeCharacter) {
         return withFeature(Feature.allowBackslashEscapeCharacter, allowBackslashEscapeCharacter);
     }
 
+    /** Controls tagged dollar quotes; false preserves dollar-containing identifier spellings. */
+    public P withDollarQuotedStringTags(boolean allowDollarQuotedStringTags) {
+        return withFeature(Feature.allowDollarQuotedStringTags, allowDollarQuotedStringTags);
+    }
+
     public P withDoubleQuotedStrings() {
         return withFeature(Feature.allowDoubleQuotedStrings, true);
     }

src/main/java/net/sf/jsqlparser/parser/feature/Feature.java

@@ -808,6 +808,12 @@ public enum Feature {
      */
     allowDoubleQuotedStrings(false),
 
+    /**
+     * Recognizes PostgreSQL $tag$...$tag$ literals. Disable for dialects where these spellings are
+     * unquoted identifiers. Untagged $$ literals are unaffected.
+     */
+    allowDollarQuotedStringTags(true),
+
     /**
      * concatenates adjacent String Literals: NEWLINE when separated by whitespace with at least one
      * newline (the SQL standard and PostgreSQL), WHITESPACE across any whitespace (GoogleSQL,

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -1664,40 +1664,44 @@ TOKEN_MGR_DECLS : {
 		return -1;
 	}
 
-    private static boolean endsWithDelimiter(Deque<Character> windowQueue, String delimiter) {
-        if (windowQueue.size() != delimiter.length()) {
-            return false;
-        }
-
-        int i = 0;
-        for (char ch : windowQueue) {
-            if (ch != delimiter.charAt(i++)) {
-                return false;
+    /** Scans a literal in linear time without tokenizing or rebuilding its whitespace. */
+    public void consumeDollarQuotedString(String closingQuote) {
+        int[] prefix = new int[closingQuote.length()];
+        for (int i = 1, matched = 0; i < closingQuote.length(); i++) {
+            while (matched > 0 && closingQuote.charAt(i) != closingQuote.charAt(matched)) {
+                matched = prefix[matched - 1];
             }
+            if (closingQuote.charAt(i) == closingQuote.charAt(matched)) {
+                matched++;
+            }
+            prefix[i] = matched;
         }
-        return true;
-    }
-
-    public void consumeDollarQuotedString(String closingQuote) {
-        Deque<Character> windowQueue = new ArrayDeque<Character>();
-        int delimiterLength = closingQuote.length();
-
         try {
-            while (true) {
+            int matched = 0;
+            while (matched < closingQuote.length()) {
                 char ch = input_stream.readChar();
-                windowQueue.addLast(ch);
-                if (windowQueue.size() > delimiterLength) {
-                    windowQueue.removeFirst();
+                while (matched > 0 && ch != closingQuote.charAt(matched)) {
+                    matched = prefix[matched - 1];
                 }
-                if (endsWithDelimiter(windowQueue, closingQuote)) {
-                    return;
+                if (ch == closingQuote.charAt(matched)) {
+                    matched++;
                 }
             }
         } catch (java.io.IOException e) {
             reportError(Math.max(closingQuote.length(), input_stream.GetImage().length()));
         }
     }
 
+    /** Rewinds any identifier suffix consumed by longest-match lexing before scanning the body. */
+    private void consumeDollarQuotedToken(Token token, String delimiter) {
+        input_stream.backup(token.image.length() - delimiter.length());
+        consumeDollarQuotedString(delimiter);
+        token.image = input_stream.GetImage();
+        token.kind = charLiteralIndex;
+        token.endLine = input_stream.getEndLine();
+        token.endColumn = input_stream.getEndColumn();
+    }
+
     /**
      * Consumes the body of a block comment after the opening delimiter has been matched,
      * honouring nesting, up to and including the outermost closing delimiter. Then backs
@@ -2407,9 +2411,7 @@ TOKEN:
 |
 <S_DOLLAR_QUOTED_STRING: "$$">
     {
-        consumeDollarQuotedString(matchedToken.image);
-        matchedToken.image = input_stream.GetImage();
-        matchedToken.kind = charLiteralIndex;
+        consumeDollarQuotedToken(matchedToken, matchedToken.image);
     }
 |
 // Bare `#` as a binary operator (PostgreSQL bitwise XOR / geometric
@@ -2420,6 +2422,13 @@ TOKEN:
 |
 <S_IDENTIFIER: (<LETTER> (<PART_LETTER>)*) | "$" | ("$" <PART_LETTER_NO_DOLLAR> (<PART_LETTER>)*)>
 {
+    if (matchedToken.image.charAt(0) == '$'
+            && Boolean.TRUE.equals(configuration.getValue(Feature.allowDollarQuotedStringTags))) {
+        String delimiter = StringValue.getDollarQuoteDelimiter(matchedToken.image);
+        if (delimiter != null) {
+            consumeDollarQuotedToken(matchedToken, delimiter);
+        }
+    }
     // MySQL `#` line comments (#2499): under the flag an unquoted identifier
     // ends at its first `#`, the rest of the line becomes a comment via the
     // stream-level substitution (real MySQL reads `42#24` as `42` plus
@@ -2428,7 +2437,7 @@ TOKEN:
     // that never opted in (the stream is only wired through the feature
     // consumers / withConfiguration); getValue avoids the String-based
     // getAsBoolean roundtrip
-    if (input_stream.featureConfiguration != null
+    if (matchedToken.kind == S_IDENTIFIER && input_stream.featureConfiguration != null
             && Boolean.TRUE.equals(configuration.getValue(Feature.allowHashLineComments))) {
         int hashIndex = matchedToken.image.indexOf('#');
         if (hashIndex > 0) {
@@ -16680,7 +16689,7 @@ List<String> captureFunctionBody() {
         tokens.add(tok.image);
     }
     foundEnd |= (tok.kind == K_END)
-        || ( tok.image.trim().startsWith("$$") && tok.image.trim().endsWith("$$")) ;
+        || (tok.kind == S_CHAR_LITERAL && StringValue.getDollarQuoteDelimiter(tok.image) != null);
 
     tok = getNextToken();
   }

src/test/java/net/sf/jsqlparser/expression/TaggedDollarStringTest.java

@@ -0,0 +1,154 @@
+/*-
+ * #%L
+ * JSQLParser library
+ * %%
+ * Copyright (C) 2004 - 2019 JSQLParser
+ * %%
+ * Dual licensed under GNU LGPL 2.1 or Apache License 2.0
+ * #L%
+ */
+package net.sf.jsqlparser.expression;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.stream.Stream;
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.parser.CCJSqlParser;
+import net.sf.jsqlparser.parser.CCJSqlParserConstants;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.parser.Token;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.Statements;
+import net.sf.jsqlparser.statement.select.PlainSelect;
+import net.sf.jsqlparser.test.TestUtils;
+import net.sf.jsqlparser.util.deparser.StatementDeParser;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class TaggedDollarStringTest {
+    static Stream<String> tags() {
+        return Stream.of("", "tag", "Tag_123", "_", "한글", "étiquette");
+    }
+
+    @ParameterizedTest
+    @MethodSource("tags")
+    void preservesLiteralBodiesAndDelimiters(String tag) throws Exception {
+        String delimiter = "$" + tag + "$";
+        for (String body : List.of("", "abc", "a\nb\r\nc\t  ", "x 'one' ''two'' \\ end",
+                "/* comment */ -- more\n#hash", "[ {\"some\":\"json\",\"with\":\"properties$\"} ]",
+                "$1 $other$ こんにちは")) {
+            String literal = delimiter + body + delimiter;
+            String sql = "SELECT " + literal + " AS value, 2 FROM t";
+            PlainSelect select = (PlainSelect) TestUtils.assertSqlCanBeParsedAndDeparsed(sql);
+            StringValue value =
+                    assertInstanceOf(StringValue.class, select.getSelectItem(0).getExpression());
+            assertEquals(body, value.getValue());
+            assertEquals(body, value.getNotExcapedValue());
+            assertEquals(delimiter, value.getQuoteStr());
+            assertEquals(literal, value.toString());
+            StringBuilder builder = new StringBuilder();
+            select.accept(new StatementDeParser(builder), null);
+            PlainSelect again = (PlainSelect) CCJSqlParserUtil.parse(builder.toString());
+            assertEquals(body, again.getSelectItem(0).getExpression(StringValue.class).getValue());
+            assertEquals(select.toString(), builder.toString());
+        }
+    }
+
+    @Test
+    void keepsDifferentTagsAndDollarSignsInsideBody() throws Exception {
+        String body = "$other$ text $Tag$ $$ $1 $t";
+        PlainSelect select =
+                (PlainSelect) CCJSqlParserUtil.parse("SELECT $tag$" + body + "$tag$::text, $1");
+        CastExpression cast = select.getSelectItem(0).getExpression(CastExpression.class);
+        assertEquals(body, ((StringValue) cast.getLeftExpression()).getValue());
+        assertInstanceOf(JdbcParameter.class, select.getSelectItem(1).getExpression());
+    }
+
+    @Test
+    void retainsIdentifiersAndSupportsOptOut() throws Exception {
+        PlainSelect select = (PlainSelect) CCJSqlParserUtil
+                .parse("SELECT $parameter, foo$bar, \"$tag$abc$tag$\", $1 FROM t");
+        for (int i = 0; i < 3; i++) {
+            assertInstanceOf(Column.class, select.getSelectItem(i).getExpression());
+        }
+        assertInstanceOf(JdbcParameter.class, select.getSelectItem(3).getExpression());
+        for (String identifier : List.of("$tag$abc$tag$", "$tag$identifier")) {
+            PlainSelect legacy = (PlainSelect) CCJSqlParserUtil.parse("SELECT " + identifier,
+                    parser -> parser.withDollarQuotedStringTags(false));
+            assertEquals(identifier,
+                    legacy.getSelectItem(0).getExpression(Column.class).getColumnName());
+        }
+        PlainSelect untagged = (PlainSelect) CCJSqlParserUtil.parse("SELECT $$text$$",
+                parser -> parser.withDollarQuotedStringTags(false));
+        assertEquals("text", untagged.getSelectItem(0).getExpression(StringValue.class).getValue());
+    }
+
+    @Test
+    void retainsBodyWithOtherLexerOptions() throws Exception {
+        PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(
+                "SELECT $t$#hash\n\\text't$tag$ \"q\"$t$",
+                parser -> parser
+                        .withDialect(net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect.MYSQL));
+        assertEquals("#hash\n\\text't$tag$ \"q\"",
+                select.getSelectItem(0).getExpression(StringValue.class).getValue());
+    }
+
+    @Test
+    void keepsLineColumnAndAbsoluteTokenPositions() {
+        String literal = "$tag$a\nb$tag$";
+        CCJSqlParser parser = CCJSqlParserUtil.newParser("SELECT " + literal + ", 2");
+        parser.getNextToken();
+        Token value = parser.getNextToken();
+        Token comma = parser.getNextToken();
+        assertEquals(CCJSqlParserConstants.S_CHAR_LITERAL, value.kind);
+        assertEquals(literal, value.image);
+        assertEquals(1, value.beginLine);
+        assertEquals(8, value.beginColumn);
+        assertEquals(2, value.endLine);
+        assertEquals(6, value.endColumn);
+        assertEquals(8, value.absoluteBegin);
+        assertEquals(8 + literal.length(), value.absoluteEnd);
+        assertEquals(value.absoluteEnd, comma.absoluteBegin);
+        assertEquals(7, comma.beginColumn);
+    }
+
+    @Test
+    void recognizesFunctionBodyAndFollowingStatement() throws Exception {
+        String body = "SELECT 'a;''b'::text;\n";
+        String sql =
+                "CREATE FUNCTION f() RETURNS text AS $fn$" + body + "$fn$ LANGUAGE SQL; SELECT 42;";
+        Statements statements = CCJSqlParserUtil.parseStatements(sql);
+        assertEquals(2, statements.size());
+        assertEquals("SELECT 42", statements.get(1).toString());
+        org.junit.jupiter.api.Assertions
+                .assertTrue(statements.get(0).toString().contains("$fn$" + body + "$fn$"));
+        assertEquals(2, CCJSqlParserUtil.parseStatements(statements.toString()).size());
+    }
+
+    @Test
+    @Timeout(10)
+    void handlesLongBodiesAndOverlappingDelimiterPrefixes() throws Exception {
+        String body = "$ta$tagX $tagtagX\n".repeat(12000);
+        String sql = "SELECT $tagtag$" + body + "$tagtag$";
+        PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(new StringReader(sql));
+        assertEquals(body, select.getSelectItem(0).getExpression(StringValue.class).getValue());
+        PlainSelect streamed = (PlainSelect) CCJSqlParserUtil.parse(
+                new java.io.ByteArrayInputStream(sql.getBytes(StandardCharsets.UTF_8)), "UTF-8");
+        assertEquals(body, streamed.getSelectItem(0).getExpression(StringValue.class).getValue());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"SELECT $tag$missing", "SELECT $Tag$wrong$tag$", "SELECT $t$ends$t",
+            "SELECT $a$text$b$", "SELECT $$missing"})
+    void rejectsUnterminatedOrMismatchedTags(String sql) {
+        assertThrows(JSQLParserException.class,
+                () -> CCJSqlParserUtil.parse(sql, parser -> parser.withTimeOut(1000)));
+    }
+}

src/test/java/net/sf/jsqlparser/statement/select/PostgresTest.java

@@ -108,8 +108,6 @@ void testNextValueIssue1863() throws JSQLParserException {
     }
 
     @Test
-    @Disabled
-    // wip
     void testDollarQuotedText() throws JSQLParserException {
         String sqlStr = "SELECT $tag$This\nis\na\nselect\ntest\n$tag$ from dual where a=b";
         PlainSelect st = (PlainSelect) CCJSqlParserUtil.parse(sqlStr);