diff --git a/.buildkite/commands/sync-localization.sh b/.buildkite/commands/sync-localization.sh new file mode 100755 index 000000000..5fb8cfc13 --- /dev/null +++ b/.buildkite/commands/sync-localization.sh @@ -0,0 +1,20 @@ +#!/bin/bash -euo pipefail + +echo "--- :git: Checking out the current branch" +BRANCH="${BUILDKITE_BRANCH:-trunk}" +git checkout "${BRANCH}" +git pull origin "${BRANCH}" + +echo '--- :robot_face: Use bot for Git operations' +source use-bot-for-git + +echo "--- :rubygems: Setting up Gems" +install_gems + +echo "--- :globe_with_meridians: :arrow_up: Generate the source language PO file for GlotPress based on `wp_localization/localization/en-US/main.ftl`" +bundle exec fastlane generate_source_po_file commit_changes:true + +echo "--- :globe_with_meridians: :arrow_down: Download and update translations from GlotPress and update the local Fluent files" +bundle exec fastlane download_translations commit_changes:true + +git push origin diff --git a/.buildkite/nightly.yml b/.buildkite/nightly.yml index 17a1cb90b..a2ae73a9a 100644 --- a/.buildkite/nightly.yml +++ b/.buildkite/nightly.yml @@ -3,12 +3,25 @@ steps: command: | echo "--- :rust: Testing" make test-rust-integration-wordpress-org-api + notify: + - slack: + channels: + - "#wordpress-rs" + message: "Nightly build failed." + if: build.state == "failed" env: TEST_ALL_PLUGINS: true -notify: - - slack: - channels: - - "#wordpress-rs" - message: "Nightly build." - if: build.state == "failed" + - label: ":globe_with_meridians: :arrow_up: :arrow_down: Sync localization" + command: .buildkite/commands/sync-localization.sh + plugins: [$CI_TOOLKIT] + notify: + - slack: + channels: + - "#wordpress-rs" + message: "Localization sync failed." + if: build.state == "failed" + env: + IMAGE_ID: $IMAGE_ID + agents: + queue: mac diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 725db099d..40cd15fd7 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -178,9 +178,9 @@ end # # The resulting PO file is saved as the source file (.pot) for translations and is synced to GlotPress. # -# @param commit_and_push_changes [Boolean] Whether to commit and push the generated PO file (default: false) +# @param commit_changes [Boolean] Whether to commit the generated PO file (default: false) # -lane :generate_source_po_file do |commit_and_push_changes: false| +lane :generate_source_po_file do |commit_changes: false| UI.header('🔄 Converting English Fluent file to PO format') FileUtils.mkdir_p(File.dirname(LOCALIZATION_PO_SOURCE_FILE)) @@ -201,7 +201,7 @@ lane :generate_source_po_file do |commit_and_push_changes: false| UI.error("❌ Failed to convert English Fluent file: #{e.message}") end - if commit_and_push_changes + if commit_changes commit_changed_files( files: LOCALIZATION_PO_SOURCE_FILE, message: 'Update source PO file (en-US.pot) to be synced to GlotPress' @@ -212,11 +212,11 @@ end # Downloads the latest translations from GlotPress and updates Fluent files # # This lane fetches translated PO files from GlotPress for all supported locales, -# converts them back to Fluent format, and optionally commits and pushes the changes. +# converts them back to Fluent format, and optionally commits the changes. # -# @param commit_and_push_changes [Boolean] Whether to commit and push the updated Fluent files (default: false) +# @param commit_changes [Boolean] Whether to commit the updated Fluent files (default: false) # -lane :download_translations do |commit_and_push_changes: false| +lane :download_translations do |commit_changes: false| UI.header('🌐 Downloading translations from GlotPress') Dir.mktmpdir do |temp_download_dir| @@ -239,7 +239,7 @@ lane :download_translations do |commit_and_push_changes: false| UI.success("✅ Updated Fluent files: #{updated_fluent_files.length} locales") - if commit_and_push_changes && updated_fluent_files.any? + if commit_changes && updated_fluent_files.any? commit_changed_files( files: updated_fluent_files, message: 'Update generated Fluent files based on latest GlotPress translations' @@ -344,7 +344,7 @@ def remove_lane_context_values(names) names.each { |name| lane_context.delete(name) } end -def commit_changed_files(files:, message:, push: true) +def commit_changed_files(files:, message:, push: false) git_add(path: files) result = git_commit( path: files, diff --git a/wp_localization/README.md b/wp_localization/README.md new file mode 100644 index 000000000..17ae5db1f --- /dev/null +++ b/wp_localization/README.md @@ -0,0 +1,62 @@ +# Localization + +This directory contains the localization files for `wordpress-rs`, enabling a translation workflow between Fluent localization files (`.ftl`) and GlotPress. + +We use GlotPress as a platform/service to have translators translate our strings from English to other languages. +But GlotPress does not support Rust's Fluent `.ftl` format as an input/output file format. + +To circumvent that, during the localization automation process, we convert the `localization/en-US/main.ftl` file to the PO format in `glopress/en-US.pot` so that a cron job can then later pick up that `.pot` file and upload it to GlotPress and send it to translators. +This transformation is handled by `bundle exec fastlane generate_source_po_file`. + +Later we then download the translations from GlotPress (which are exported in the `.po` format), then regenerate the `localization/*/main.ftl` files for each language based on those downloaded translated `.po` files. +This is handled by `bundle exec fastlane download_translations` + +_Note that we have to commit the `glotpress/en-US.pot` file here because that file is picked up by a cron job in our systems on a regular basis (as opposed to the `.pot` being sent via API on demand) given how those imports are integrated in our systems._ + +## Workflow for Developers + +### Automated Daily Sync + +The localization process runs automatically every night via a Buildkite pipeline (`.buildkite/nightly.yml`). This automated job: + +1. **Generates the source en-US PO file** from `localization/en-US/main.ftl` and commits it to `glotpress/en-US.pot` +2. **Downloads latest translations** from GlotPress for all supported locales as PO files +3. **Converts downloaded PO files to Fluent format** and updates the corresponding `localization/*/main.ftl` files +4. **Commits the updated translation files** to the repository + +This means that translation updates happen automatically without developer intervention. + +### Adding New Localization Strings + +1. **Add strings to the source file**: Edit `wp_localization/localization/en-US/main.ftl` to add or update localization strings. + +2. **The automated nightly job will handle the rest**: The next nightly run will automatically: + - Convert your changes to PO format (`glotpress/en-US.pot`) + - Commit the updated source file + - Upload to GlotPress via the wpcom cron job + +### Manual Operations (Optional) + +If you need to run the localization sync manually instead of waiting for the nightly job: + +**Generate source PO file:** +```bash +bundle exec fastlane generate_source_po_file +``` + +**Download GlotPress translations and generate local Fluent translation files:** +```bash +bundle exec fastlane download_translations +``` + +### Helper Lanes + +- **`download_po_files_from_glotpress`**: Downloads PO files for all supported locales +- **`generate_fluent_file_from_po`**: Converts individual PO files back to Fluent format + +## References + +- **Fluent format**: Uses [Project Fluent](https://projectfluent.org/) for localization files +- **PO format**: Standard [gettext](https://www.gnu.org/software/gettext/manual/gettext.html) format used by GlotPress +- **Conversion tool**: Uses the [fluent-tools](https://github.com/Automattic/fluent-rust-tools) Ruby gem for format conversion +- **GlotPress integration**: Downloads translations from `https://translate.wordpress.com/projects/mobile/wordpress-rs`