Skip to content

Commit 24f05da

Browse files
snazysnazy
authored
Nano syntax highlighter fails to parse pre-installed json.nanorc (#1157)
Fix parsing of system default nano syntax highlighting files This change addresses issues with parsing system default `/usr/share/nano/*.nanorc` files in JLine's NanorcParser. The parser was failing to handle certain regular expressions in these files due to differences between POSIX and Java regex syntax. Key improvements: - Added `posixToJavaRegex` function to translate POSIX regex patterns to Java-compatible patterns - Implemented proper handling of bracket expressions, escape sequences, and POSIX character classes - Fixed handling of special cases like `[^][]`, `[.f\]`, and POSIX classes like `[[:word:]]` - Added comprehensive test cases to verify regex translations - Improved error handling by catching PatternSyntaxException to gracefully handle invalid patterns - Refactored and simplified the regex handling code in the final commit The implementation handles these specific differences: - First `]` in bracket expressions (unescaped in POSIX, needs `\]` in Java) - `[` in bracket expressions (unescaped in POSIX, needs `\[` in Java) - Backslashes before closing brackets (needs double escaping in Java) - POSIX character classes (translated to Java equivalents) This fix enables JLine to properly parse and highlight syntax in system-provided nanorc files. Fixes #1156
1 parent 0030f35 commit 24f05da

3 files changed

Lines changed: 881 additions & 54 deletions

File tree

‎builtins/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@
4747
<artifactId>junit-jupiter-api</artifactId>
4848
<scope>test</scope>
4949
</dependency>
50+
<dependency>
51+
<groupId>org.junit.jupiter</groupId>
52+
<artifactId>junit-jupiter-params</artifactId>
53+
<scope>test</scope>
54+
</dependency>
5055
</dependencies>
5156

5257
<build>

‎builtins/src/main/java/org/jline/builtins/SyntaxHighlighter.java‎

Lines changed: 187 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,7 @@ private AttributedStringBuilder _highlight(
356356
return asb;
357357
}
358358

359-
private static class HighlightRule {
359+
static class HighlightRule {
360360
public enum RuleType {
361361
PATTERN,
362362
START_END,
@@ -465,7 +465,7 @@ public String toString() {
465465
}
466466
}
467467

468-
private static class NanorcParser {
468+
static class NanorcParser {
469469
private static final String DEFAULT_SYNTAX = "default";
470470
private final String name;
471471
private final String target;
@@ -497,7 +497,7 @@ public void parse() throws IOException {
497497
idx++;
498498
line = line.trim();
499499
if (!line.isEmpty() && !line.startsWith("#")) {
500-
List<String> parts = RuleSplitter.split(fixRegexes(line));
500+
List<String> parts = RuleSplitter.split(line);
501501
if (parts.get(0).equals("syntax")) {
502502
syntaxName = parts.get(1);
503503
List<Pattern> filePatterns = new ArrayList<>();
@@ -547,7 +547,7 @@ public void parse() throws IOException {
547547
}
548548
for (String l : colorTheme.get(key).split("\\\\n")) {
549549
idx++;
550-
addHighlightRule(RuleSplitter.split(fixRegexes(l)), idx, key);
550+
addHighlightRule(RuleSplitter.split(l), idx, key);
551551
}
552552
}
553553
} else {
@@ -560,7 +560,7 @@ public void parse() throws IOException {
560560
if (theme != null) {
561561
for (String l : theme.split("\\\\n")) {
562562
idx++;
563-
addHighlightRule(RuleSplitter.split(fixRegexes(l)), idx, TOKEN_NANORC);
563+
addHighlightRule(RuleSplitter.split(l), idx, TOKEN_NANORC);
564564
}
565565
} else {
566566
Log.warn("Unknown token type: ", key);
@@ -573,23 +573,6 @@ public void parse() throws IOException {
573573
}
574574
}
575575

576-
private String fixRegexes(String line) {
577-
return line.replaceAll("\\\\<", "\\\\b")
578-
.replaceAll("\\\\>", "\\\\b")
579-
.replaceAll("\\[:alnum:]", "\\\\p{Alnum}")
580-
.replaceAll("\\[:alpha:]", "\\\\p{Alpha}")
581-
.replaceAll("\\[:blank:]", "\\\\p{Blank}")
582-
.replaceAll("\\[:cntrl:]", "\\\\p{Cntrl}")
583-
.replaceAll("\\[:digit:]", "\\\\p{Digit}")
584-
.replaceAll("\\[:graph:]", "\\\\p{Graph}")
585-
.replaceAll("\\[:lower:]", "\\\\p{Lower}")
586-
.replaceAll("\\[:print:]", "\\\\p{Print}")
587-
.replaceAll("\\[:punct:]", "\\\\p{Punct}")
588-
.replaceAll("\\[:space:]", "\\\\s")
589-
.replaceAll("\\[:upper:]", "\\\\p{Upper}")
590-
.replaceAll("\\[:xdigit:]", "\\\\p{XDigit}");
591-
}
592-
593576
private boolean addHighlightRule(List<String> parts, int idx, String tokenName) {
594577
boolean out = true;
595578
if (parts.get(0).equals("color")) {
@@ -656,43 +639,50 @@ private void addHighlightRule(String reference, List<String> parts, boolean case
656639
Styles.StyleCompiler sh = new Styles.StyleCompiler(spec, true);
657640
AttributedStyle style = new StyleResolver(sh::getStyle).resolve("." + reference);
658641

659-
if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.PATTERN) {
660-
if (parts.size() == 2) {
661-
highlightRules.get(tokenName).add(new HighlightRule(style, doPattern(".*", caseInsensitive)));
662-
} else {
663-
for (int i = 2; i < parts.size(); i++) {
664-
highlightRules
665-
.get(tokenName)
666-
.add(new HighlightRule(style, doPattern(parts.get(i), caseInsensitive)));
642+
try {
643+
if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.PATTERN) {
644+
if (parts.size() == 2) {
645+
highlightRules.get(tokenName).add(new HighlightRule(style, doPattern(".*", caseInsensitive)));
646+
} else {
647+
for (int i = 2; i < parts.size(); i++) {
648+
highlightRules
649+
.get(tokenName)
650+
.add(new HighlightRule(style, doPattern(parts.get(i), caseInsensitive)));
651+
}
667652
}
653+
} else if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.START_END) {
654+
String s = parts.get(2);
655+
String e = parts.get(3);
656+
highlightRules
657+
.get(tokenName)
658+
.add(new HighlightRule(
659+
style,
660+
doPattern(s.substring(7, s.length() - 1), caseInsensitive),
661+
doPattern(e.substring(5, e.length() - 1), caseInsensitive)));
662+
} else if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.PARSER_START_WITH) {
663+
highlightRules
664+
.get(tokenName)
665+
.add(new HighlightRule(
666+
HighlightRule.RuleType.PARSER_START_WITH,
667+
style,
668+
parts.get(2).substring(10)));
669+
} else if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.PARSER_CONTINUE_AS) {
670+
highlightRules
671+
.get(tokenName)
672+
.add(new HighlightRule(
673+
HighlightRule.RuleType.PARSER_CONTINUE_AS,
674+
style,
675+
parts.get(2).substring(11)));
668676
}
669-
} else if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.START_END) {
670-
String s = parts.get(2);
671-
String e = parts.get(3);
672-
highlightRules
673-
.get(tokenName)
674-
.add(new HighlightRule(
675-
style,
676-
doPattern(s.substring(7, s.length() - 1), caseInsensitive),
677-
doPattern(e.substring(5, e.length() - 1), caseInsensitive)));
678-
} else if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.PARSER_START_WITH) {
679-
highlightRules
680-
.get(tokenName)
681-
.add(new HighlightRule(
682-
HighlightRule.RuleType.PARSER_START_WITH,
683-
style,
684-
parts.get(2).substring(10)));
685-
} else if (HighlightRule.evalRuleType(parts) == HighlightRule.RuleType.PARSER_CONTINUE_AS) {
686-
highlightRules
687-
.get(tokenName)
688-
.add(new HighlightRule(
689-
HighlightRule.RuleType.PARSER_CONTINUE_AS,
690-
style,
691-
parts.get(2).substring(11)));
677+
} catch (PatternSyntaxException e) {
678+
Log.warn("Invalid highlight regex", reference, parts, e);
679+
} catch (Exception e) {
680+
Log.warn("Failure while handling highlight regex", reference, parts, e);
692681
}
693682
}
694683

695684
private Pattern doPattern(String regex, boolean caseInsensitive) {
685+
regex = Parser.fixRegexes(regex);
696686
return caseInsensitive ? Pattern.compile(regex, Pattern.CASE_INSENSITIVE) : Pattern.compile(regex);
697687
}
698688
}
@@ -796,7 +786,7 @@ public int getEnd() {
796786
}
797787
}
798788

799-
private static class Parser {
789+
static class Parser {
800790
private static final char escapeChar = '\\';
801791
private String blockCommentTokenName;
802792
private BlockCommentDelimiters blockCommentDelimiters;
@@ -980,5 +970,148 @@ private boolean isEscaped(final CharSequence buffer, final int pos) {
980970
}
981971
return isEscapeChar(buffer, pos - 1);
982972
}
973+
974+
/**
975+
* Perform Posix/Java regex fixups. This function parses the given regex and escapes according to these rules:
976+
*
977+
* <p>The first {@code ]} in a bracket expression does not need to be escaped in Posix,translate to {@code \]}.
978+
*
979+
* <p>Same as above for a negating bracket expression like {@code [^][]}, translate to {@code [^\]\[]}.
980+
*
981+
* <p>Any {@code [} in a bracket expression does not need to be escaped in Posix, translate to {@code \[}.
982+
*
983+
* <p>Any {@code ]} not in a bracket expression is valid in both Posix and Java, no translation.
984+
*
985+
* <p>A backslash before the closing bracket like {@code [.f\]} is not an escape of the closing bracket,
986+
* the backslash needs to be escaped for Java, translate to {@code [.f\\]}.
987+
*
988+
* <p>Do not perform the above translations within an escape via {@code \}, except for {@code \<} and {@code \>} to {@code \b}.
989+
*
990+
* <p>Replace the Posix classes like {@code [:space:]} or {@code [:digit:]} to Java classes, inside and outside a bracket expression.
991+
*
992+
* @param posix Posix regex
993+
* @return Java regex
994+
*/
995+
static String fixRegexes(String posix) {
996+
int len = posix.length();
997+
StringBuilder java = new StringBuilder();
998+
999+
boolean inBracketExpression = false;
1000+
1001+
int i = 0;
1002+
char next;
1003+
try {
1004+
for (; i < len; i++) {
1005+
char c = posix.charAt(i);
1006+
1007+
switch (c) {
1008+
case escapeChar:
1009+
next = posix.charAt(++i);
1010+
// Don't translate anything after the \ character escape
1011+
if (inBracketExpression && next == ']') {
1012+
inBracketExpression = false;
1013+
java.append("\\\\").append(next);
1014+
} else {
1015+
// Translate '\<' and '\>' to '\b'
1016+
if (next == '<' || next == '>') {
1017+
next = 'b';
1018+
}
1019+
java.append(c).append(next);
1020+
}
1021+
break;
1022+
case '[':
1023+
if (i == len - 1) {
1024+
throw new IllegalArgumentException("Lone [ at the end of (index " + i + "): " + posix);
1025+
}
1026+
// Handle "single bracket" Posix "classes" like [:space:] or [:digit:]
1027+
if (posix.charAt(i + 1) == ':') {
1028+
int afterClass = nextAfterClass(posix, i + 2);
1029+
if (!posix.regionMatches(afterClass, ":]", 0, 2)) {
1030+
java.append("[:");
1031+
i++;
1032+
inBracketExpression = true;
1033+
} else {
1034+
String className = posix.substring(i + 2, afterClass);
1035+
java.append(replaceClass(className));
1036+
i = afterClass + 1;
1037+
}
1038+
break;
1039+
}
1040+
if (inBracketExpression) {
1041+
// Translate lone [ to \[
1042+
java.append('\\').append(c);
1043+
} else {
1044+
inBracketExpression = true;
1045+
java.append(c);
1046+
next = posix.charAt(i + 1);
1047+
if (next == ']') {
1048+
i++;
1049+
java.append("\\]");
1050+
} else if (next == '^' && posix.charAt(i + 2) == ']') {
1051+
i += 2;
1052+
java.append("^\\]");
1053+
}
1054+
}
1055+
break;
1056+
case ']':
1057+
inBracketExpression = false;
1058+
java.append(c);
1059+
break;
1060+
default:
1061+
java.append(c);
1062+
break;
1063+
}
1064+
}
1065+
} catch (Exception e) {
1066+
throw new IllegalArgumentException(
1067+
"Posix-to-Java regex translation failed around index " + i + " of: " + posix, e);
1068+
}
1069+
1070+
return java.toString();
1071+
}
1072+
1073+
private static String replaceClass(String className) {
1074+
switch (className) {
1075+
case "alnum":
1076+
return "\\p{Alnum}";
1077+
case "alpha":
1078+
return "\\p{Alpha}";
1079+
case "blank":
1080+
return "\\p{Blank}";
1081+
case "cntrl":
1082+
return "\\p{Cntrl}";
1083+
case "digit":
1084+
return "\\p{Digit}";
1085+
case "graph":
1086+
return "\\p{Graph}";
1087+
case "lower":
1088+
return "\\p{Lower}";
1089+
case "print":
1090+
return "\\p{Print}";
1091+
case "punct":
1092+
return "\\p{Punct}";
1093+
case "space":
1094+
return "\\s";
1095+
case "upper":
1096+
return "\\p{Upper}";
1097+
case "xdigit":
1098+
return "\\p{XDigit}";
1099+
}
1100+
throw new IllegalArgumentException("Unknown class '" + className + "'");
1101+
}
1102+
1103+
private static int nextAfterClass(String s, int idx) {
1104+
if (s.charAt(idx) == ':') {
1105+
idx++;
1106+
}
1107+
while (true) {
1108+
char c = s.charAt(idx);
1109+
if (!Character.isLetterOrDigit(c)) {
1110+
break;
1111+
}
1112+
idx++;
1113+
}
1114+
return idx;
1115+
}
9831116
}
9841117
}

0 commit comments

Comments
 (0)