Merge branch 'sicp_i18n' into i18n_gh_actions

Author: coder114514

dictionary/short.txt

@@ -0,0 +1 @@
+function: 睡觉

i18n/controllers/path.ts

@@ -1,5 +1,7 @@
-export default function PathGenerator(path: string) {
-  const pathArray: string[] = path.split(".");
+import path from "path";
+
+export default function PathGenerator(filePath: string) {
+  const pathArray: string[] = filePath.split(".");
   let gitPath: string = "";
   const gitPathPrefix: string[] = ["chapter", "section", "subsection"];
 
@@ -15,5 +17,5 @@ export default function PathGenerator(path: string) {
     i++;
   }
 
-  return gitPath + ".xml";
+  return path.resolve("../xml/en" + gitPath + ".xml");
 }

i18n/controllers/recurTranslate.ts

@@ -34,6 +34,7 @@ const MAXLEN = Number(process.env.MAX_LEN) || 3000;
 // change to true to avoid calling openai api, useful for troubleshooting
 // chunking logic
 const troubleshoot = false;
+let languageNames = new Intl.DisplayNames(["en"], { type: "language" });
 
 // Centralized logging to prevent duplicate messages
 const errorMessages = new Set();
@@ -73,21 +74,29 @@ export function getFileErrors(): Record<
 const createParser = () =>
   (sax as any).createStream(false, { trim: false }, { strictEntities: true });
 
-async function translate(language: string, filePath: string): Promise<void> {
+async function translate(langCode: string, filePath: string): Promise<void> {
   const startTime = new Date().getTime();
   let assistant;
 
   try {
     // Use the provided file path directly without modification
     const input_path = filePath;
+    const language = languageNames.of(langCode);
+
+    if (language === undefined) {
+      throw new Error('Undefined language');
+    }
 
     if (!troubleshoot) assistant = await createAssistant(language, ai as any);
 
+    
     // Generate output path by replacing "/en/" with "/cn/" in the path
     const output_path = filePath.replace(
       path.sep + "en" + path.sep,
-      path.sep + "cn" + path.sep
+      path.sep + langCode + path.sep
     );
+    
+
 
     const translated: string = await recursivelyTranslate(
       language,
@@ -112,7 +121,7 @@ async function translate(language: string, filePath: string): Promise<void> {
       });
     }
     const elapsed = new Date().getTime() - startTime;
-    console.log(filePath + " took " + elapsed / 1000.0 + " seconds");
+    console.log("Took " + elapsed / 1000.0 + " seconds");
   }
 }
 
@@ -273,7 +282,7 @@ async function recursivelyTranslate(
 
   try {
     await new Promise<void>((resolve, reject) => {
-      console.log("Translating " + filePath + " at " + thread.id);
+      console.log("Translating " + filePath + " to " + language + " at " + thread.id);
       // Variables to track current depth and segments.
       let currentDepth = 0;
       let currentSegment = "";
@@ -413,7 +422,7 @@ async function recursivelyTranslate(
     try {
       await ai.beta.threads.messages.create(thread.id, {
         role: "user",
-        content: `<TRANSLATE> ${chunk} </TRANSLATE>`
+        content: `Translate to ${language}: <TRANSLATE> ${chunk} </TRANSLATE>`
       });
 
       const run = await ai.beta.threads.runs.createAndPoll(thread.id, {

i18n/index.ts

@@ -1,15 +1,23 @@
 import PathGenerator from "./controllers/path.ts";
 import translate, { getFileErrors } from "./controllers/recurTranslate.ts";
 import fs from "fs";
+import util from "util";
 import path from "path";
 import { fileURLToPath } from "url";
 import { dirname } from "path";
 import OpenAI from "openai";
+import { permission } from "process";
 
 // Get the directory name of the current module
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
+const readdir = util.promisify(fs.readdir);
+const getDirectories = async source =>
+  (await readdir(source, { withFileTypes: true }))
+    .filter(dirent => dirent.isDirectory())
+    .map(dirent => dirent.name);
+
 // Global variables for tracking translation state
 // These need to be accessible by signal handlers
 let xmlFiles: string[] = [];
@@ -52,7 +60,7 @@ async function saveSummaryLog() {
           failedDel.push(file.id);
         }
       })
-    ).then(() => console.log("successfully deleted all files"))
+    ).then(() => console.log("successfully deleted all files"));
 
     const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
     let summaryLog = `
@@ -189,11 +197,11 @@ async function findAllXmlFiles(directory: string): Promise<string[]> {
 }
 
 // Function to check if a file needs translation
-async function needsTranslation(enFilePath: string): Promise<boolean> {
+async function needsTranslation(enFilePath: string, lang: string): Promise<boolean> {
   // Generate the corresponding cn file path
   const cnFilePath = enFilePath.replace(
     path.sep + "en" + path.sep,
-    path.sep + "cn" + path.sep
+    path.sep + lang + path.sep
   );
 
   try {
@@ -212,9 +220,9 @@ async function needsTranslation(enFilePath: string): Promise<boolean> {
   }
 }
 
-export default async function fancyName(path: string) {
+export default async function fancyName(path: string, language: string) {
   const fullPath = PathGenerator(path);
-  await translate("Chinese", fullPath);
+  await translate(language, fullPath);
 }
 
 // use "all" to translate every xml
@@ -223,6 +231,20 @@ export default async function fancyName(path: string) {
   await setupCleanupHandlers();
 
   try {
+    if ((process.argv[2], process.argv[3])) {
+      fancyName(process.argv[2], process.argv[3]);
+      return;
+    }
+
+    let languages: string[] = [];
+
+    if (process.argv[2] === "all") {
+      languages = await getDirectories(path.join(__dirname, "../xml"));
+      console.dir(languages);
+    } else {
+      languages.push(process.argv[2]);
+    }
+
     // Get the absolute path to the xml/en directory using proper path resolution
     const enDirPath = path.resolve(__dirname, "../xml/en");
 
@@ -234,19 +256,20 @@ export default async function fancyName(path: string) {
     }
 
     console.log(`Found ${xmlFiles.length} XML files to check for translation`);
-
+   
+    for (const lang of languages) {
     // Filter files that need translation
     filesToTranslate = [];
     for (const file of xmlFiles) {
-      if (await needsTranslation(file)) {
+      if (await needsTranslation(file, lang)) {
         filesToTranslate.push(file as never);
       }
     }
 
     console.log(`${filesToTranslate.length} files need translation`);
 
     if (filesToTranslate.length === 0) {
-      console.log("No files need translation. Exiting.");
+      console.log(`No files need translation for ${lang}.`);
       return;
     }
 
@@ -267,7 +290,7 @@ export default async function fancyName(path: string) {
         batch.map(async file => {
           try {
             console.log(`Starting translation for ${file}`);
-            await translate("Chinese", file);
+            await translate(lang, file);
             return { file, success: true };
           } catch (error) {
             // Return failure with error but don't log yet
@@ -329,6 +352,7 @@ export default async function fancyName(path: string) {
 
     // Save a detailed summary to a log file
     await saveSummaryLog();
+  }
   } catch (e) {
     console.error("Error during translation process:", e);
   }

i18n/initializers/initialize.ts

@@ -11,6 +11,7 @@ export default async function createAssistant(language: string, ai: OpenAI) {
   const assistant = await ai.beta.assistants.create({
     name: "SICP Translator",
     instructions: `You are a professional translator with high technical skills in computer science.
+                You always translate all provided tags to ${language}
                 You MUST adhere to the following rules strictly:
                 1. ALWAYS use the exact translations for technical terms found in the uploaded reference file.
                 2. If a term appears in the reference file, you MUST use the provided translation without exception.
@@ -22,7 +23,7 @@ export default async function createAssistant(language: string, ai: OpenAI) {
     tools: [{ type: "file_search" }]
   });
 
-  const fileStreams = [path.resolve(__dirname, "../../dictionary/cn.txt")].map(
+  const fileStreams = [path.resolve(__dirname, "../../dictionary/short.txt")].map(
     path => fs.createReadStream(path)
   );