diff --git a/Dockerfile b/Dockerfile index b4efb17d6102..78cd652cc815 100644 --- a/Dockerfile +++ b/Dockerfile @@ -155,7 +155,7 @@ COPY --chown=olympia:olympia locale ${LOCALE_DIR} RUN \ --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \ --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \ - --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \ + --mount=type=bind,source=scripts/compile_locales.py,target=${HOME}/scripts/compile_locales.py \ make -f Makefile-docker compile_locales # More efficient caching by mounting the exact files we need @@ -175,10 +175,10 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/ RUN \ --mount=type=bind,src=src,target=${HOME}/src \ --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \ + --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \ --mount=type=bind,src=manage.py,target=${HOME}/manage.py \ < settings_local.py -DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets +make -f Makefile-docker update_assets EOF FROM base AS production diff --git a/Makefile-docker b/Makefile-docker index 87c2ef8c6bc2..91ff05f2d683 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -72,14 +72,8 @@ data_load: ./manage.py data_load $(ARGS) .PHONY: update_assets -update_assets: - # Copy files required in compress_assets to the static folder - # If changing this here, make sure to adapt tests in amo/test_commands.py - $(PYTHON_COMMAND) manage.py compress_assets - $(PYTHON_COMMAND) manage.py generate_jsi18n_files - # Collect static files: This MUST be run last or files will be missing - $(PYTHON_COMMAND) manage.py collectstatic --noinput - +update_assets: ## Update the static assets + $(HOME)/scripts/update_assets.py .PHONY: update_deps update_deps: ## Update the dependencies @@ -218,7 +212,7 @@ extract_locales: ## extracts and merges translation strings .PHONE: compile_locales compile_locales: ## compiles translation strings $(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/locale.txt - ./locale/compile-mo.sh ./locale/ + $(HOME)/scripts/compile_locales.py .PHONY: help_submake help_submake: diff --git a/Makefile-os b/Makefile-os index 96c888fd6433..e4dc9c6f1bdd 100644 --- a/Makefile-os +++ b/Makefile-os @@ -161,17 +161,17 @@ docker_clean_build_cache: ## Remove buildx build cache .PHONY: clean_docker clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine -.PHONY: docker_update_deps -docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target +.PHONY: docker_sync_host +docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target docker compose run \ --rm \ --no-deps \ $(DOCKER_RUN_ARGS) \ web \ - make update_deps + ./scripts/sync_host_files.py .PHONY: up_pre -up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state +up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state .PHONY: up_start up_start: docker_mysqld_volume_create ## Start the docker containers diff --git a/locale/compile-mo.sh b/locale/compile-mo.sh deleted file mode 100755 index d50a894a7f19..000000000000 --- a/locale/compile-mo.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# syntax: -# compile-mo.sh locale-dir/ - -# Make this script fail if any command exits wit exit code != 0 -set -ue - -function process_po_file() { - pofile=$1 - dir=$(dirname "$pofile") - lang=$(echo "$pofile" | cut -d "/" -f2) - stem=$(basename "$pofile" .po) - touch "${dir}/${stem}.mo" - dennis-cmd lint --errorsonly "$pofile" && msgfmt -o "${dir}/${stem}.mo" "$pofile" -} - -# We are spawning sub processes with `xargs` -# and the function needs to be available in that sub process -export -f process_po_file - -function usage() { - echo "syntax:" - echo "compile-mo.sh locale-dir/" - exit 1 -} - -# check if file and dir are there -if [[ ($# -ne 1) || (! -d "$1") ]]; then usage; fi - -# Ensure dennis-cmd cli is available in the environment -hash dennis-cmd - -echo "compiling django.po..." -find $1 -type f -name "django.po" -print0 | xargs -0 -n1 -P4 bash -c 'process_po_file "$@"' _ - -echo "compiling djangojs.po..." -find $1 -type f -name "djangojs.po" -print0 | xargs -0 -n1 -P4 bash -c 'process_po_file "$@"' _ diff --git a/scripts/compile_locales.py b/scripts/compile_locales.py new file mode 100755 index 000000000000..17e0e663cbc4 --- /dev/null +++ b/scripts/compile_locales.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import os +import subprocess +from concurrent.futures import ThreadPoolExecutor + + +def process_po_file(pofile, attempt=0): + """Process a single .po file, creating corresponding .mo file.""" + print('processing', pofile) + directory = os.path.dirname(pofile) + stem = os.path.splitext(os.path.basename(pofile))[0] + mo_path = os.path.join(directory, f'{stem}.mo') + + # Touch the .mo file + open(mo_path, 'a').close() + + try: + # Run dennis-cmd lint + subprocess.run( + ['dennis-cmd', 'lint', '--errorsonly', pofile], + capture_output=True, + check=False, + ) + # If lint passes, run msgfmt + subprocess.run(['msgfmt', '-o', mo_path, pofile], check=True) + return + except subprocess.CalledProcessError as e: + if attempt < 3: + print(f'Failed attempt {attempt} for {pofile}, retrying...') + return process_po_file(pofile, attempt=attempt + 1) + raise e + + +def main(): + # Ensure 'dennis' is installed + try: + import dennis as _ + except ImportError: + print( + 'Error: dennis is not installed. Please install it with pip install dennis' + ) + exit(1) + + locale_dir = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + '..', + 'locale', + ) + ) + + print(f'Compiling locales in {locale_dir}') + + # Collect all files first + django_files = [] + djangojs_files = [] + for root, _, files in os.walk(locale_dir): + for file in files: + if file == 'django.po': + django_files.append(os.path.join(root, file)) + elif file == 'djangojs.po': + djangojs_files.append(os.path.join(root, file)) + + # Process django.po files in parallel + with ThreadPoolExecutor() as executor: + executor.map(process_po_file, django_files + djangojs_files) + + +if __name__ == '__main__': + main() diff --git a/scripts/sync_host_files.py b/scripts/sync_host_files.py new file mode 100755 index 000000000000..37d27b40d217 --- /dev/null +++ b/scripts/sync_host_files.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +import json +import os +import shutil +import subprocess + + +def main(): + BUILD_INFO = os.environ.get('BUILD_INFO') + + subprocess.run(['make', 'update_deps'], check=True) + + HOME = os.environ.get('HOME') + + for dir in ['static-build', 'site-static']: + path = os.path.join(HOME, dir) + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(path, exist_ok=True) + + with open(BUILD_INFO, 'r') as f: + build_info = json.load(f) + + if build_info.get('target') == 'production': + subprocess.run(['make', 'compile_locales'], check=True) + subprocess.run(['make', 'update_assets'], check=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/update_assets.py b/scripts/update_assets.py new file mode 100644 index 000000000000..f9ce3125edb4 --- /dev/null +++ b/scripts/update_assets.py @@ -0,0 +1,30 @@ +import os +import subprocess + + +def main(): + script_prefix = ['python3', 'manage.py'] + + environment = os.environ.copy() + # Always run in production mode without any development settings + environment['DJANGO_SETTINGS_MODULE'] = 'olympia.lib.settings_base' + + subprocess.run( + script_prefix + ['compress_assets'], + check=True, + env=environment, + ) + subprocess.run( + script_prefix + ['generate_jsi18n_files'], + check=True, + env=environment, + ) + subprocess.run( + script_prefix + ['collectstatic', '--noinput'], + check=True, + env=environment, + ) + + +if __name__ == '__main__': + main()