diff --git a/.gitignore b/.gitignore index d2a01bd4f2c1..1303a8a5131d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,5 +55,3 @@ tmp/* # do not ignore the following files !docker-compose.private.yml !private/README.md -!storage/.gitignore -!deps/.gitkeep diff --git a/deps/.gitkeep b/deps/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/docker-compose.yml b/docker-compose.yml index 4e847cacc989..a282f242af43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,8 +47,8 @@ services: # used by: web, worker, nginx - ${HOST_MOUNT_SOURCE:?}:/data/olympia - ${HOST_MOUNT_SOURCE:?}deps:/deps - - data_site_static:/data/olympia/site-static - - ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage + - ./site-static:/data/olympia/site-static + - ./storage:/data/olympia/storage worker: <<: *olympia command: [ @@ -65,7 +65,7 @@ services: volumes: - ${HOST_MOUNT_SOURCE:?}:/data/olympia - ${HOST_MOUNT_SOURCE:?}deps:/deps - - ${HOST_MOUNT_SOURCE:?}storage:/data/olympia/storage + - ./storage:/data/olympia/storage extra_hosts: - "olympia.test:127.0.0.1" restart: on-failure:5 @@ -94,18 +94,16 @@ services: retries: 3 start_interval: 1s volumes: - # Don't mount generated files. They only exist in the container - # and would otherwiser be deleted by mounting the cwd volume above - - data_static_build:/data/olympia/static-build - - data_site_static:/data/olympia/site-static + - ./static-build:/data/olympia/static-build + - ./site-static:/data/olympia/site-static nginx: image: nginx volumes: - data_nginx:/etc/nginx/conf.d - ${HOST_MOUNT_SOURCE:?}:/srv - - data_site_static:/srv/site-static - - ${HOST_MOUNT_SOURCE:?}storage:/srv/storage + - ./site-static:/srv/site-static + - ./storage:/srv/storage ports: - "80:80" networks: @@ -204,10 +202,6 @@ networks: enable_ipv6: false volumes: - # Volumes for static files that should not be - # mounted from the host. - data_static_build: - data_site_static: # Volumes for the production olympia mounts # allowing to conditionally mount directories # from the host or from the image to @@ -218,7 +212,6 @@ volumes: # (data_olympia_):/ data_olympia_: data_olympia_deps: - data_olympia_storage: # Volume for rabbitmq/redis to avoid anonymous volumes data_rabbitmq: data_redis: diff --git a/scripts/setup.py b/scripts/setup.py index 176a05237c51..45a7d4dd21b8 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -3,8 +3,13 @@ import os +root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + +env_path = os.path.join(root, '.env') + + def set_env_file(values): - with open('.env', 'w') as f: + with open(env_path, 'w') as f: print('Environment:') for key, value in values.items(): f.write(f'{key}="{value}"\n') @@ -14,8 +19,8 @@ def set_env_file(values): def get_env_file(): env = {} - if os.path.exists('.env'): - with open('.env', 'r') as f: + if os.path.exists(env_path): + with open(env_path, 'r') as f: for line in f: key, value = line.strip().split('=', 1) env[key] = value.strip('"') @@ -143,6 +148,10 @@ def main(): } ) + # Create the directories that are expected to exist in the container. + for dir in ['deps', 'site-static', 'static-build', 'storage']: + os.makedirs(os.path.join(root, dir), exist_ok=True) + if __name__ == '__main__': main() diff --git a/src/olympia/core/apps.py b/src/olympia/core/apps.py index 0a066020137e..9d878f84781e 100644 --- a/src/olympia/core/apps.py +++ b/src/olympia/core/apps.py @@ -165,10 +165,12 @@ def nginx_check(app_configs, **kwargs): configs = [ (settings.MEDIA_ROOT, 'http://nginx/user-media'), - (settings.STATIC_ROOT, 'http://nginx/static'), (settings.STATIC_FILES_PATH, 'http://nginx/static'), + (settings.STATIC_ROOT, 'http://nginx/static'), ] + files_to_remove = [] + for dir, base_url in configs: file_path = os.path.join(dir, 'test.txt') file_url = f'{base_url}/test.txt' @@ -177,12 +179,15 @@ def nginx_check(app_configs, **kwargs): errors.append(Error(f'{dir} does not exist', id='setup.E007')) try: - open(file_path, 'w').close() + with open(file_path, 'w') as f: + f.write(dir) + + files_to_remove.append(file_path) response = requests.get(file_url) expected_config = ( (response.status_code, 200), - (response.text, ''), + (response.text, dir), (response.headers.get('X-Served-By'), 'nginx'), ) @@ -197,9 +202,10 @@ def nginx_check(app_configs, **kwargs): id='setup.E010', ) ) - finally: - if os.path.exists(file_path): - os.remove(file_path) + + # Always remove the files we created. + for file_path in files_to_remove: + os.remove(file_path) return errors diff --git a/src/olympia/core/tests/test_apps.py b/src/olympia/core/tests/test_apps.py index fe0641cf6fd3..8c6823aeca15 100644 --- a/src/olympia/core/tests/test_apps.py +++ b/src/olympia/core/tests/test_apps.py @@ -193,7 +193,7 @@ def _test_nginx_response( expected_config = ( (status_code, 200), - (response_text, ''), + (response_text, self.media_root), (served_by, 'nginx'), ) diff --git a/storage/.gitignore b/storage/.gitignore deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/make/make.spec.js b/tests/make/make.spec.js index 8c3abc779c5d..f177e9cf6174 100644 --- a/tests/make/make.spec.js +++ b/tests/make/make.spec.js @@ -105,9 +105,7 @@ describe('docker-compose.yml', () => { target: '/data/olympia', }), expect.objectContaining({ - source: isProdMountTarget - ? 'data_olympia_storage' - : expect.any(String), + source: expect.any(String), target: '/data/olympia/storage', }), ]), @@ -125,11 +123,11 @@ describe('docker-compose.yml', () => { expect(web.volumes).toEqual( expect.arrayContaining([ expect.objectContaining({ - source: 'data_static_build', + source: expect.any(String), target: '/data/olympia/static-build', }), expect.objectContaining({ - source: 'data_site_static', + source: expect.any(String), target: '/data/olympia/site-static', }), ]), @@ -162,14 +160,12 @@ describe('docker-compose.yml', () => { target: '/srv', }), expect.objectContaining({ - source: 'data_site_static', + source: expect.any(String), target: '/srv/site-static', }), // mapping for local host directory to /data/olympia/storage expect.objectContaining({ - source: isProdMountTarget - ? 'data_olympia_storage' - : expect.any(String), + source: expect.any(String), target: '/srv/storage', }), ]), diff --git a/tests/make/test_setup.py b/tests/make/test_setup.py index 6656f64d66e3..d4d94b559e3c 100644 --- a/tests/make/test_setup.py +++ b/tests/make/test_setup.py @@ -229,3 +229,15 @@ def test_override_env_olympia_deps_development_on_target_development(self): def test_olympia_deps_override(self): main() self.assert_set_env_file_called_with(OLYMPIA_DEPS='test') + + +@override_env() +@mock.patch('scripts.setup.os.makedirs') +def test_make_dirs(mock_makedirs): + from scripts.setup import root + + main() + assert mock_makedirs.call_args_list == [ + mock.call(os.path.join(root, dir), exist_ok=True) + for dir in ['deps', 'site-static', 'static-build', 'storage'] + ]