handle utf surrogate character literals (#438)

resolves #405

Author: Greg Adams

rewrite-java-11/src/main/java/org/openrewrite/java/Java11ParserVisitor.java

@@ -689,13 +689,26 @@ public J visitLambdaExpression(LambdaExpressionTree node, Space fmt) {
         return new J.Lambda(randomId(), fmt, Markers.EMPTY, params, arrow, body, type(node));
     }
 
+    public final static int SURR_FIRST = 0xD800;
+    public final static int SURR_LAST = 0xDFFF;
+
     @Override
     public J visitLiteral(LiteralTree node, Space fmt) {
         cursor(endPos(node));
         Object value = node.getValue();
         JavaType.Primitive type = primitive(((JCTree.JCLiteral) node).typetag);
+        if (value instanceof Character) {
+            char c = (Character) value;
+            if (c >= SURR_FIRST && c <= SURR_LAST) {
+                String valueSource = source.substring(((JCLiteral) node).getStartPosition(), endPos(node));
+                int escapeIndex = valueSource.indexOf("\\u");
+                return new J.Literal(randomId(), fmt, Markers.EMPTY, null, null,
+                        new J.Literal.ModifiedUtf8Surrogate(valueSource.substring(0, escapeIndex),
+                                valueSource.substring(escapeIndex)), type);
+            }
+        }
         return new J.Literal(randomId(), fmt, Markers.EMPTY, value,
-                source.substring(((JCLiteral) node).getStartPosition(), endPos(node)), type);
+                source.substring(((JCLiteral) node).getStartPosition(), endPos(node)), null, type);
     }
 
     @Override

rewrite-java/src/main/java/org/openrewrite/java/JavaPrinter.java

@@ -690,7 +690,12 @@ public J visitLambda(Lambda lambda, P p) {
     public J visitLiteral(Literal literal, P p) {
         visitSpace(literal.getPrefix(), Space.Location.LITERAL_PREFIX, p);
         StringBuilder acc = getPrinter();
-        acc.append(literal.getValueSource());
+        Literal.ModifiedUtf8Surrogate modifiedUtf8Surrogate = literal.getModifiedUtf8Surrogate();
+        if (modifiedUtf8Surrogate != null) {
+            acc.append(modifiedUtf8Surrogate.getEscapeSequence()).append(modifiedUtf8Surrogate.getCodePoint());
+        } else {
+            acc.append(literal.getValueSource());
+        }
         return literal;
     }
 

rewrite-java/src/main/java/org/openrewrite/java/tree/J.java

@@ -2612,6 +2612,7 @@ public Parameters withParams(List<JRightPadded<J>> parameters) {
     @EqualsAndHashCode(callSuper = false, onlyExplicitlyIncluded = true)
     @Data
     final class Literal implements J, Expression {
+
         @EqualsAndHashCode.Include
         UUID id;
 
@@ -2626,8 +2627,13 @@ final class Literal implements J, Expression {
         Object value;
 
         @With
+        @Nullable
         String valueSource;
 
+        @With
+        @Nullable
+        ModifiedUtf8Surrogate modifiedUtf8Surrogate;
+
         /**
          * Including String literals
          */
@@ -2640,7 +2646,7 @@ public Literal withType(@Nullable JavaType type) {
                 return this;
             }
             if (type instanceof JavaType.Primitive) {
-                return new Literal(id, prefix, markers, value, valueSource, (JavaType.Primitive) type);
+                return new Literal(id, prefix, markers, value, valueSource, modifiedUtf8Surrogate, (JavaType.Primitive) type);
             }
             return this;
         }
@@ -2655,23 +2661,19 @@ public Coordinates.Literal getCoordinates() {
             return new Coordinates.Literal(this);
         }
 
-        public <T> String transformValue(Function<T, Object> transform) {
-            Matcher valueMatcher = Pattern.compile("(.*)" + Pattern.quote(value == null ? "null" : value.toString()) + "(.*)")
-                    .matcher(printTrimmed().replace("\\", ""));
-            if (valueMatcher.find()) {
-                String prefix = valueMatcher.group(1);
-                String suffix = valueMatcher.group(2);
-
-                //noinspection unchecked
-                return prefix + transform.apply((T) value) + suffix;
-            }
-            throw new IllegalStateException("Encountered a literal `" + this + "` that could not be transformed");
-        }
-
         @Override
         public String toString() {
             return "Literal(" + LiteralToString.toString(this) + ")";
         }
+
+        @Value
+        public static class ModifiedUtf8Surrogate {
+            @With
+            String escapeSequence;
+
+            @With
+            String codePoint;
+        }
     }
 
     @ToString

rewrite-test/src/main/kotlin/org/openrewrite/java/search/SemanticallyEqualTest.kt

@@ -284,6 +284,7 @@ interface SemanticallyEqualTest {
                     Markers.EMPTY,
                     true,
                     "true",
+                    null,
                     JavaType.Primitive.Boolean
                 ),
                 Markers.EMPTY
@@ -348,6 +349,7 @@ interface SemanticallyEqualTest {
                         Markers.EMPTY,
                         0,
                         "0",
+                        null,
                         JavaType.Primitive.Int
                     )
                 )
@@ -363,6 +365,7 @@ interface SemanticallyEqualTest {
                         Markers.EMPTY,
                         "thisString",
                         "thisString",
+                        null,
                         JavaType.Primitive.String
                     )
                 )
@@ -378,6 +381,7 @@ interface SemanticallyEqualTest {
                         Markers.EMPTY,
                         null,
                         "null",
+                        null,
                         JavaType.Primitive.String
                     )
                 )
@@ -393,6 +397,7 @@ interface SemanticallyEqualTest {
                         Markers.EMPTY,
                         0,
                         "0",
+                        null,
                         JavaType.Primitive.Int
                     )
                 )

rewrite-test/src/main/kotlin/org/openrewrite/java/tree/MethodDeclarationTest.kt

@@ -100,8 +100,7 @@ interface MethodDeclarationTest : JavaTreeTest {
     }
 
     @Test()
-    @Disabled("Issue #405")
-    fun unicodeCharacterLiterals(jp: JavaParser) = assertParsePrintAndProcess(jp, CompilationUnit,
+    fun modifiedUtf8SurrogateCharacterLiterals(jp: JavaParser) = assertParsePrintAndProcess(jp, CompilationUnit,
             """
                 public class A {
                     private boolean isSockJsSpecialChar(char ch) {