feat: expand replacement table and change dash mapping to single hyphen

- Change en/em dash replacement from '--' to '-'
- Add hyphen variants: U+2010 HYPHEN, U+2011 NON-BREAKING HYPHEN,
  U+2012 FIGURE DASH, U+2015 HORIZONTAL BAR, U+FE58 SMALL EM DASH
- Add soft hyphen (U+00AD) removal (invisible layout hint)
- Add bullets: U+2022 BULLET, U+2023 TRIANGULAR BULLET -> *,
  U+2043 HYPHEN BULLET -> -
- Add dot leaders: U+2024 -> ., U+2025 -> ..
- Add arrows: U+2190 -> <-, U+2192 -> ->, U+2191 -> ^, U+2193 -> v
- Add math operators: U+00D7 -> x, U+00F7 -> /, U+2044 -> /

Author: mit-d

CHANGELOG.md

@@ -18,6 +18,10 @@
 
 ### Changed
 
+- En dash and em dash now replace to `-` instead of `--`
+- Expanded `--fix` replacement table with: hyphen variants (U+2010-2012, U+2015,
+  U+FE58), soft hyphen (removed), bullets, dot leaders, arrows (`->`, `<-`, `^`,
+  `v`), and math operators (`x`, `/`)
 - Add `pytest-sugar` for improved test output
 - Replace mypy with [ty](https://github.com/astral-sh/ty) for type checking
 - Move dev dependencies from `optional-dependencies` to `dependency-groups`

docs/check-unicode.1

@@ -317,19 +317,49 @@ Lines already flushed remain in the output.
 .B Smart quotes
 \(lq\(rq \(oq\(cq and variants \(-> replaced with ASCII quotes
 .TP
-.B Dashes
-Em dash (U+2014), en dash (U+2013), minus sign (U+2212) \(-> replaced with
-.B \-\-
-or
+.B Dashes and hyphens
+Em dash, en dash, figure dash, horizontal bar, minus sign, and other
+dash\-like characters \(-> replaced with
 .BR \- .
 .TP
+.B Soft hyphen
+U+00AD \(-> removed (invisible layout hint, not content).
+.TP
 .B Fancy spaces
 Non\-breaking space, em space, thin space, and 14 other Unicode space characters
 \(-> replaced with a regular space.
 .TP
 .B Ellipsis
 Horizontal ellipsis (U+2026) \(-> replaced with
 .BR ... .
+.TP
+.B Bullets
+Bullet (U+2022), triangular bullet, hyphen bullet \(-> replaced with
+.B *
+or
+.BR \- .
+.TP
+.B Dot leaders
+One dot leader, two dot leader \(-> replaced with
+.B .
+or
+.BR .. .
+.TP
+.B Arrows
+\(-> and \(<- \(-> replaced with
+.B \->
+and
+.BR <\- ;
+\(ua and \(da \(-> replaced with
+.B ^
+and
+.BR v .
+.TP
+.B Math operators
+Multiplication sign (\(mu) \(-> replaced with
+.BR x ;
+division sign (\(di) and fraction slash \(-> replaced with
+.BR / .
 .
 .SS Dangerous invisible characters (never auto\-fixed)
 .TP

src/check_unicode/categories.py

@@ -36,11 +36,17 @@
     0x201F: '"',  # DOUBLE HIGH-REVERSED-9 QUOTATION MARK
     0x00AB: '"',  # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
     0x00BB: '"',  # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
-    # Dashes
-    0x2013: "--",  # EN DASH
-    0x2014: "--",  # EM DASH
-    # Minus
+    # Dashes and hyphens
+    0x2010: "-",  # HYPHEN
+    0x2011: "-",  # NON-BREAKING HYPHEN
+    0x2012: "-",  # FIGURE DASH
+    0x2013: "-",  # EN DASH
+    0x2014: "-",  # EM DASH
+    0x2015: "-",  # HORIZONTAL BAR
     0x2212: "-",  # MINUS SIGN
+    0xFE58: "-",  # SMALL EM DASH
+    # Soft hyphen (invisible layout hint, not content)
+    0x00AD: "",  # SOFT HYPHEN
     # Fancy spaces -> regular space
     0x00A0: " ",  # NO-BREAK SPACE
     0x2000: " ",  # EN QUAD
@@ -57,4 +63,20 @@
     0x3000: " ",  # IDEOGRAPHIC SPACE
     # Ellipsis
     0x2026: "...",  # HORIZONTAL ELLIPSIS
+    # Bullets
+    0x2022: "*",  # BULLET
+    0x2023: "*",  # TRIANGULAR BULLET
+    0x2043: "-",  # HYPHEN BULLET
+    # Dot leaders
+    0x2024: ".",  # ONE DOT LEADER
+    0x2025: "..",  # TWO DOT LEADER
+    # Arrows
+    0x2190: "<-",  # LEFTWARDS ARROW
+    0x2192: "->",  # RIGHTWARDS ARROW
+    0x2191: "^",  # UPWARDS ARROW
+    0x2193: "v",  # DOWNWARDS ARROW
+    # Math operators
+    0x00D7: "x",  # MULTIPLICATION SIGN
+    0x00F7: "/",  # DIVISION SIGN
+    0x2044: "/",  # FRACTION SLASH
 }

tests/test_cli.py

@@ -121,7 +121,7 @@ def test_fix_multiple_files_all_fixed(self, tmp_path: Path) -> None:
         f2.write_text("word\u2014word\n", encoding="utf-8")
         assert main(["--fix", str(f1), str(f2)]) == 1
         assert f1.read_text(encoding="utf-8") == 'He said "hello"\n'
-        assert f2.read_text(encoding="utf-8") == "word--word\n"
+        assert f2.read_text(encoding="utf-8") == "word-word\n"
 
     def test_fix_dangerous_still_reported(self, tmp_path: Path) -> None:
         """Fix mode does not remove dangerous characters."""

tests/test_fixer.py

@@ -116,15 +116,33 @@ class TestFixText:
             ("\u201equote\u201f", '"quote"'),
             ("\u00abguillemet\u00bb", '"guillemet"'),
             ("\u2039angle\u203a", "'angle'"),
-            ("word\u2014word", "word--word"),
-            ("1\u20132", "1--2"),
+            ("word\u2014word", "word-word"),
+            ("1\u20132", "1-2"),
             ("x \u2212 y", "x - y"),
             ("hello\u00a0world", "hello world"),
             ("a\u2003b", "a b"),
             ("a\u2009b", "a b"),
             ("a\u200ab", "a b"),
             ("a\u3000b", "a b"),
             ("wait\u2026", "wait..."),
+            ("a\u2010b", "a-b"),
+            ("a\u2011b", "a-b"),
+            ("a\u2012b", "a-b"),
+            ("a\u2015b", "a-b"),
+            ("a\ufe58b", "a-b"),
+            ("soft\u00adhyphen", "softhyphen"),
+            ("\u2022 item", "* item"),
+            ("\u2023 item", "* item"),
+            ("\u2043 item", "- item"),
+            ("ch\u20241", "ch.1"),
+            ("ch\u20251", "ch..1"),
+            ("a \u2192 b", "a -> b"),
+            ("b \u2190 a", "b <- a"),
+            ("\u2191up", "^up"),
+            ("\u2193down", "vdown"),
+            ("2 \u00d7 3", "2 x 3"),
+            ("6 \u00f7 2", "6 / 2"),
+            ("1\u20442", "1/2"),
         ],
         ids=[
             "smart-double-quotes",
@@ -143,6 +161,24 @@ class TestFixText:
             "hair-space",
             "ideographic-space",
             "ellipsis",
+            "hyphen",
+            "non-breaking-hyphen",
+            "figure-dash",
+            "horizontal-bar",
+            "small-em-dash",
+            "soft-hyphen",
+            "bullet",
+            "triangular-bullet",
+            "hyphen-bullet",
+            "one-dot-leader",
+            "two-dot-leader",
+            "right-arrow",
+            "left-arrow",
+            "up-arrow",
+            "down-arrow",
+            "multiplication-sign",
+            "division-sign",
+            "fraction-slash",
         ],
     )
     def test_fix_replaces_character(self, input_text: str, expected: str) -> None:
@@ -169,7 +205,7 @@ def test_mixed_fixable_nonfixable_dangerous(self) -> None:
     def test_multiline_text(self) -> None:
         """fix_text handles multi-line strings correctly."""
         text = "line1 \u201chi\u201d\nline2 word\u2014word\nline3 wait\u2026\n"
-        expected = 'line1 "hi"\nline2 word--word\nline3 wait...\n'
+        expected = 'line1 "hi"\nline2 word-word\nline3 wait...\n'
         assert fix_text(text) == expected
 
 
@@ -182,8 +218,8 @@ class TestFixFileReplacements:
             ("\u201chello\u201d", '"hello"'),
             ("It\u2019s", "It's"),
             ("\u2018word\u2019", "'word'"),
-            ("word\u2014word", "word--word"),
-            ("1\u20132", "1--2"),
+            ("word\u2014word", "word-word"),
+            ("1\u20132", "1-2"),
             ("x \u2212 y", "x - y"),
             ("hello\u00a0world", "hello world"),
             ("a\u2003b", "a b"),