Fix UnicodeDecodeError in the top-level tooling on non-UTF-8 locales (#299)

The tooling called read_text()/write_text() with no encoding=, so Python fell back to the locale encoding. Four model LICENSE files contain UTF-8 curly quotes whose 0x9d byte is undefined in cp1252, so `make gallery` and the regenerate-license pre-commit hook crashed on Windows.

Author: yurekami

format_xml.py

@@ -170,14 +170,14 @@ def main() -> int:
 
   failed: list[pathlib.Path] = []
   for p in args.paths:
-    text = p.read_text()
+    text = p.read_text(encoding='utf-8')
     formatted = format_xml(text)
     if args.check:
       if text != formatted:
         failed.append(p)
     elif args.write:
       if text != formatted:
-        p.write_text(formatted)
+        p.write_text(formatted, encoding='utf-8')
     else:
       sys.stdout.write(formatted)
 

generate_gallery.py

@@ -76,7 +76,13 @@ def display_name(robot):
   maker = robot.split('/')[0]
   readme = pathlib.Path(f'{maker}/README.md')
   if readme.exists():
-    title = readme.read_text().splitlines()[0].strip().lstrip('#').strip()
+    title = (
+      readme.read_text(encoding='utf-8')
+      .splitlines()[0]
+      .strip()
+      .lstrip('#')
+      .strip()
+    )
     title = _README_TITLE_SUFFIX.sub('', title).rstrip()
     if title:
       return title
@@ -382,7 +388,7 @@ def sort_func(xml):
 
 def detect_license(license_path):
   """Identify the SPDX license name from the LICENSE file contents."""
-  text = pathlib.Path(license_path).read_text()
+  text = pathlib.Path(license_path).read_text(encoding='utf-8')
   lower = text.lower()
   if 'apache license' in lower and 'version 2' in lower:
     return 'Apache-2.0'
@@ -441,7 +447,7 @@ def write_gallery_to_readme(rendered, dofs, readme_path='README.md'):
     sections.append(table)
 
   body = '\n\n'.join(sections)
-  readme = pathlib.Path(readme_path).read_text()
+  readme = pathlib.Path(readme_path).read_text(encoding='utf-8')
   pattern = re.compile(
     re.escape(MODELS_BEGIN) + r'.*?' + re.escape(MODELS_END), re.DOTALL
   )
@@ -450,7 +456,7 @@ def write_gallery_to_readme(rendered, dofs, readme_path='README.md'):
       f'models markers not found in {readme_path}; expected\n  {MODELS_BEGIN}\n  {MODELS_END}'
     )
   new = pattern.sub(f'{MODELS_BEGIN}\n\n{body}\n\n{MODELS_END}', readme)
-  pathlib.Path(readme_path).write_text(new)
+  pathlib.Path(readme_path).write_text(new, encoding='utf-8')
 
 
 def main(argv):

regenerate_license.py

@@ -45,12 +45,12 @@ def get_base_license(root: pathlib.Path) -> str:
   """
   opensource_license = root / 'opensource' / 'LICENSE'
   if opensource_license.exists():
-    return opensource_license.read_text()
+    return opensource_license.read_text(encoding='utf-8')
 
   # Fall back to extracting from the existing concatenated LICENSE.
   existing = root / 'LICENSE'
   if existing.exists():
-    sections = existing.read_text().split(HLINE + '\n')
+    sections = existing.read_text(encoding='utf-8').split(HLINE + '\n')
     if sections:
       return sections[-1]
 
@@ -84,7 +84,7 @@ def generate_license(root: pathlib.Path) -> str:
     out += HLINE
     out += f"License for contents in the directory '{lf.parent.name}/'\n"
     out += HLINE + '\n'
-    out += lf.read_text() + '\n\n'
+    out += lf.read_text(encoding='utf-8') + '\n\n'
 
   out += HLINE
   out += 'The following license applies to all other contents\n'
@@ -115,7 +115,7 @@ def main():
       print('FAIL: LICENSE file does not exist.', file=sys.stderr)
       sys.exit(1)
 
-    current = license_path.read_text()
+    current = license_path.read_text(encoding='utf-8')
     if current != generated:
       print(
         'FAIL: LICENSE file is out of date. '
@@ -127,7 +127,7 @@ def main():
     print('OK: LICENSE file is up to date.')
     return
 
-  license_path.write_text(generated)
+  license_path.write_text(generated, encoding='utf-8')
   print(f'LICENSE file regenerated at {license_path}')
 
 

test/model_dir_test.py

@@ -84,7 +84,9 @@ class ContributorsTest(absltest.TestCase):
   """Checks CONTRIBUTORS.md sections are sorted alphabetically by first name."""
 
   def test_sorted(self) -> None:
-    contributors = (_ROOT_DIR / 'CONTRIBUTORS.md').read_text().splitlines()
+    contributors = (
+      (_ROOT_DIR / 'CONTRIBUTORS.md').read_text(encoding='utf-8').splitlines()
+    )
 
     # Each section is a contiguous block of lines starting with "- ".
     section_start = None