diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml index 1619f498eb95..dea9fdd64908 100644 --- a/.github/actions/run-docker/action.yml +++ b/.github/actions/run-docker/action.yml @@ -23,10 +23,6 @@ inputs: description: 'Docker target to run (development|production)' required: false default: 'production' - mount: - description: 'Mount host files at runtime? (development|production)' - required: false - default: 'production' deps: description: 'Which dependencies to install at runtime? (development|production)' required: false @@ -44,7 +40,6 @@ runs: DOCKER_DIGEST="${{ inputs.digest }}" \ DOCKER_TARGET="${{ inputs.target }}" \ OLYMPIA_UID="$(id -u)" \ - OLYMPIA_MOUNT="${{ inputs.mount }}" \ OLYMPIA_DEPS="${{ inputs.deps }}" \ DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \ DOCKER_WAIT="true" diff --git a/.github/workflows/_test_check.yml b/.github/workflows/_test_check.yml index eebd0f900289..88ec6f90ecb6 100644 --- a/.github/workflows/_test_check.yml +++ b/.github/workflows/_test_check.yml @@ -46,7 +46,6 @@ jobs: name: | version: '${{ matrix.version }}' | target: '${{ matrix.target }}' | - mount: '${{ matrix.mount }}' | deps: '${{ matrix.deps }}' strategy: fail-fast: false @@ -57,9 +56,6 @@ jobs: target: - development - production - mount: - - development - - production deps: - development - production @@ -72,7 +68,6 @@ jobs: Values passed to the action: version: ${{ matrix.version }} target: ${{ matrix.target }} - mount: ${{ matrix.mount }} deps: ${{ matrix.deps }} EOF - name: ${{ matrix.version == 'local' && 'Uncached Build' || 'Pull' }} Check @@ -84,7 +79,6 @@ jobs: with: version: ${{ matrix.version }} target: ${{ matrix.target }} - mount: ${{ matrix.mount }} deps: ${{ matrix.deps }} run: make check - name: Cached Build Check @@ -93,7 +87,6 @@ jobs: with: version: ${{ matrix.version }} target: ${{ matrix.target }} - mount: ${{ matrix.mount }} deps: ${{ matrix.deps }} run: echo true diff --git a/.github/workflows/_test_main.yml b/.github/workflows/_test_main.yml index 38457e047a11..cd69c91de696 100644 --- a/.github/workflows/_test_main.yml +++ b/.github/workflows/_test_main.yml @@ -88,7 +88,6 @@ jobs: services: '' digest: ${{ inputs.digest }} version: ${{ inputs.version }} - mount: development deps: development run: | split="--splits ${{ needs.test_config.outputs.splits }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d782354e3346..743afda17c0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,6 @@ jobs: with: digest: ${{ needs.build.outputs.digest }} version: ${{ needs.build.outputs.version }} - mount: development deps: development target: development run: | @@ -140,7 +139,6 @@ jobs: with: digest: ${{ needs.build.outputs.digest }} version: ${{ needs.build.outputs.version }} - mount: development deps: development run: make extract_locales diff --git a/.gitignore b/.gitignore index d2a01bd4f2c1..8bf5971cf22d 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,5 @@ tmp/* !private/README.md !storage/.gitignore !deps/.gitkeep +!site-static/.gitkeep +!static-build/.gitkeep diff --git a/Dockerfile b/Dockerfile index ff4bd52bbab2..06c019eb800b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -148,6 +148,9 @@ EOF FROM base AS development +# Copy build info from info +COPY --from=info ${BUILD_INFO} ${BUILD_INFO} + FROM base AS locales ARG LOCALE_DIR=${HOME}/locale # Compile locales @@ -157,7 +160,7 @@ COPY --chown=olympia:olympia locale ${LOCALE_DIR} RUN \ --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \ --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \ - --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \ + --mount=type=bind,source=scripts/compile_locales.py,target=${HOME}/scripts/compile_locales.py \ make -f Makefile-docker compile_locales # More efficient caching by mounting the exact files we need @@ -175,12 +178,12 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/ # Run that command without having to copy the whole source code # This will shave nearly 1 minute off the best case build time RUN \ - --mount=type=bind,src=src,target=${HOME}/src \ --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \ + --mount=type=bind,src=src,target=${HOME}/src \ + --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \ --mount=type=bind,src=manage.py,target=${HOME}/manage.py \ < settings_local.py -DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets +make -f Makefile-docker update_assets EOF FROM base AS production diff --git a/Makefile-docker b/Makefile-docker index 2630b30e2902..2039d0d36a4e 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -45,14 +45,8 @@ data_load: ./manage.py data_load $(ARGS) .PHONY: update_assets -update_assets: - # Copy files required in compress_assets to the static folder - # If changing this here, make sure to adapt tests in amo/test_commands.py - $(PYTHON_COMMAND) manage.py compress_assets - $(PYTHON_COMMAND) manage.py generate_jsi18n_files - # Collect static files: This MUST be run last or files will be missing - $(PYTHON_COMMAND) manage.py collectstatic --noinput - +update_assets: ## Update the static assets + $(HOME)/scripts/update_assets.py .PHONY: update_deps update_deps: ## Update the dependencies @@ -191,7 +185,7 @@ extract_locales: ## extracts and merges translation strings .PHONE: compile_locales compile_locales: ## compiles translation strings $(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/locale.txt - ./locale/compile-mo.sh ./locale/ + $(HOME)/scripts/compile_locales.py .PHONY: help_submake help_submake: diff --git a/Makefile-os b/Makefile-os index e3079503fcd3..15fbe78fd8f7 100644 --- a/Makefile-os +++ b/Makefile-os @@ -153,21 +153,22 @@ docker_clean_build_cache: ## Remove buildx build cache .PHONY: clean_docker clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine -.PHONY: docker_update_deps -docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target +.PHONY: docker_sync_host +docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target docker compose run \ --rm \ --no-deps \ $(DOCKER_RUN_ARGS) \ web \ - make update_deps + ./scripts/sync_host_files.py .PHONY: up_pre -up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state +up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state .PHONY: up_start up_start: docker_mysqld_volume_create ## Start the docker containers docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS) + docker compose rm olympia_volumes -f .PHONY: up_post up_post: docker_clean_images docker_clean_volumes ## Post-up the environment, setup files, volumes and host state diff --git a/docker-compose.yml b/docker-compose.yml index 8671855728e6..7e4caaaf81fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,7 +45,7 @@ services: command: ["sleep", "infinity"] volumes: # used by: web, worker, nginx - - ${HOST_MOUNT_SOURCE:?}:/data/olympia + - ./:/data/olympia worker: <<: *olympia command: [ @@ -60,7 +60,7 @@ services: "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO", ] volumes: - - ${HOST_MOUNT_SOURCE:?}:/data/olympia + - ./:/data/olympia extra_hosts: - "olympia.test:127.0.0.1" restart: on-failure:5 @@ -93,7 +93,7 @@ services: image: nginx volumes: - data_nginx:/etc/nginx/conf.d - - ${HOST_MOUNT_SOURCE:?}:/srv + - ./:/srv ports: - "80:80" networks: @@ -192,15 +192,6 @@ networks: enable_ipv6: false volumes: - # Volumes for the production olympia mounts - # allowing to conditionally mount directories - # from the host or from the image to - # in the running docker container. - # If OLYMPIA_MOUNT_SOURCE matches (data_olympia_) - # then we use the production volume mounts. Otherwise - # it will map to the current directory ./ - # (data_olympia_):/ - data_olympia_: # Volume for rabbitmq/redis to avoid anonymous volumes data_rabbitmq: data_redis: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 5dfe6ad22e0e..06aca156b3c0 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -30,12 +30,6 @@ fi NEW_HOST_UID=$(get_olympia_uid) OLYMPIA_ID_STRING="${NEW_HOST_UID}:$(get_olympia_gid)" -# If we are on production mode, update the ownership of /data/olympia to match the new id -if [[ "${HOST_MOUNT}" == "production" ]]; then - echo "Updating ownership of ${HOME} to ${OLYMPIA_ID_STRING}" - chown -R ${OLYMPIA_ID_STRING} ${HOME} -fi - cat < { describe.each( permutations({ DOCKER_TARGET: ['development', 'production'], - OLYMPIA_MOUNT: ['development', 'production'], DOCKER_VERSION: ['local', 'latest'], }), )('\n%s\n', (config) => { - const { DOCKER_TARGET, OLYMPIA_MOUNT, DOCKER_VERSION } = config; - const isProdTarget = DOCKER_TARGET === 'production'; - const isProdMount = OLYMPIA_MOUNT === 'production'; - const isProdMountTarget = isProdMount && isProdTarget; + const { DOCKER_TARGET, DOCKER_VERSION } = config; const inputValues = { DOCKER_TARGET, - OLYMPIA_MOUNT, DOCKER_VERSION, DEBUG: 'debug', DATA_BACKUP_SKIP: 'skip', @@ -118,17 +113,13 @@ describe('docker-compose.yml', () => { expect(service.volumes).toEqual( expect.arrayContaining([ expect.objectContaining({ - source: isProdMountTarget ? 'data_olympia_' : expect.any(String), + source: expect.any(String), target: '/data/olympia', }), ]), ); - const { - OLYMPIA_MOUNT, - DOCKER_VERSION, - DOCKER_TARGET, - ...environmentOutput - } = inputValues; + const { DOCKER_VERSION, DOCKER_TARGET, ...environmentOutput } = + inputValues; expect(service.environment).toEqual( expect.objectContaining({ ...environmentOutput, @@ -163,7 +154,7 @@ describe('docker-compose.yml', () => { }), // mapping for /data/olympia/ directory to /srv expect.objectContaining({ - source: isProdMountTarget ? 'data_olympia_' : expect.any(String), + source: expect.any(String), target: '/srv', }), ]), @@ -263,7 +254,6 @@ describe('docker-compose.yml', () => { const ignoreKeys = [ // Ignored because these values are explicitly mapped to the host_* values 'OLYMPIA_UID', - 'OLYMPIA_MOUNT', // Ignored because the HOST_UID is always set to the host user's UID 'HOST_UID', ]; diff --git a/tests/make/test_setup.py b/tests/make/test_setup.py index 97d5b82fdc5b..47f4a9a97df0 100644 --- a/tests/make/test_setup.py +++ b/tests/make/test_setup.py @@ -10,8 +10,6 @@ 'DOCKER_TAG', 'DOCKER_TARGET', 'HOST_UID', - 'HOST_MOUNT', - 'HOST_MOUNT_SOURCE', 'OLYMPIA_DEPS', 'DEBUG', ] @@ -184,44 +182,6 @@ def test_debug_override(self): self.assert_set_env_file_called_with(DEBUG='test') -@override_env() -class TestHostMount(BaseTestClass): - def test_default_host_mount(self): - main() - self.assert_set_env_file_called_with( - HOST_MOUNT='development', HOST_MOUNT_SOURCE='./' - ) - - @override_env(DOCKER_TARGET='production') - def test_host_mount_set_by_docker_target(self): - main() - self.assert_set_env_file_called_with( - HOST_MOUNT='production', HOST_MOUNT_SOURCE='data_olympia_' - ) - - @override_env(DOCKER_TARGET='production', OLYMPIA_MOUNT='test') - def test_invalid_host_mount_set_by_env_ignored(self): - main() - self.assert_set_env_file_called_with( - HOST_MOUNT='production', HOST_MOUNT_SOURCE='data_olympia_' - ) - - @override_env(DOCKER_TARGET='development', OLYMPIA_MOUNT='production') - def test_host_mount_overriden_by_development_target(self): - main() - self.assert_set_env_file_called_with( - HOST_MOUNT='development', HOST_MOUNT_SOURCE='./' - ) - - @override_env(DOCKER_TARGET='production') - def test_host_mount_from_file_ignored(self): - self.mock_get_env_file.return_value = {'HOST_MOUNT': 'development'} - main() - self.assert_set_env_file_called_with( - HOST_MOUNT='production', HOST_MOUNT_SOURCE='data_olympia_' - ) - - @override_env() class TestOlympiaDeps(BaseTestClass): def test_default_olympia_deps(self):