diff --git a/scripts/compile_locales.py b/scripts/compile_locales.py index 17e0e663cbc4..23df4783915c 100755 --- a/scripts/compile_locales.py +++ b/scripts/compile_locales.py @@ -3,17 +3,16 @@ import os import subprocess from concurrent.futures import ThreadPoolExecutor +from pathlib import Path -def process_po_file(pofile, attempt=0): +def process_po_file(pofile, attempt=1): """Process a single .po file, creating corresponding .mo file.""" - print('processing', pofile) - directory = os.path.dirname(pofile) - stem = os.path.splitext(os.path.basename(pofile))[0] - mo_path = os.path.join(directory, f'{stem}.mo') + pofile_path = Path(pofile) + print('processing', pofile_path.as_posix()) + mo_path = pofile_path.with_suffix('.mo') - # Touch the .mo file - open(mo_path, 'a').close() + mo_path.touch() try: # Run dennis-cmd lint @@ -32,35 +31,25 @@ def process_po_file(pofile, attempt=0): raise e -def main(): +def compile_locales(): # Ensure 'dennis' is installed - try: - import dennis as _ - except ImportError: - print( - 'Error: dennis is not installed. Please install it with pip install dennis' - ) - exit(1) + import dennis as _dennis # type: ignore # noqa: F401 - locale_dir = os.path.abspath( - os.path.join( - os.path.dirname(__file__), - '..', - 'locale', - ) - ) + HOME = os.environ.get('HOME') + + locale_dir = Path(HOME) / 'locale' print(f'Compiling locales in {locale_dir}') # Collect all files first django_files = [] djangojs_files = [] - for root, _, files in os.walk(locale_dir): + for root, _, files in locale_dir.walk(): for file in files: if file == 'django.po': - django_files.append(os.path.join(root, file)) + django_files.append(root / file) elif file == 'djangojs.po': - djangojs_files.append(os.path.join(root, file)) + djangojs_files.append(root / file) # Process django.po files in parallel with ThreadPoolExecutor() as executor: @@ -68,4 +57,4 @@ def main(): if __name__ == '__main__': - main() + compile_locales() diff --git a/scripts/sync_host_files.py b/scripts/sync_host_files.py index 6ca2154bceef..503600e20ab1 100755 --- a/scripts/sync_host_files.py +++ b/scripts/sync_host_files.py @@ -5,7 +5,7 @@ import subprocess -def main(): +def sync_host_files(): BUILD_INFO = os.environ.get('BUILD_INFO') subprocess.run(['make', 'update_deps'], check=True) @@ -19,4 +19,4 @@ def main(): if __name__ == '__main__': - main() + sync_host_files() diff --git a/scripts/update_assets.py b/scripts/update_assets.py index 91c02ba9178a..8f6b6c8d157d 100755 --- a/scripts/update_assets.py +++ b/scripts/update_assets.py @@ -1,24 +1,31 @@ #!/usr/bin/env python3 +import argparse import os import shutil import subprocess +from pathlib import Path -def main(): +def clean_static_dirs(verbose: bool = False): HOME = os.environ.get('HOME') STATIC_DIRS = ['static-build', 'site-static'] - for dir in STATIC_DIRS: - path = os.path.join(HOME, dir) - os.makedirs(path, exist_ok=True) - for file in os.listdir(path): - file_path = os.path.join(path, file) - print(f'Removing {file_path}') - if os.path.isdir(file_path): - shutil.rmtree(file_path) + for directory in STATIC_DIRS: + path = Path(HOME) / directory + path.mkdir(parents=True, exist_ok=True) + for entry in path.iterdir(): + entry_path = entry.as_posix() + if verbose: + print(f'Removing {entry_path}') + if entry.is_dir(): + shutil.rmtree(entry_path) else: - os.remove(file_path) + os.remove(entry_path) + + +def update_assets(verbose: bool = False): + clean_static_dirs(verbose) script_prefix = ['python3', 'manage.py'] @@ -44,4 +51,7 @@ def main(): if __name__ == '__main__': - main() + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', action='store_true') + args = parser.parse_args() + update_assets(args.verbose) diff --git a/tests/make/test_compile_locales.py b/tests/make/test_compile_locales.py new file mode 100644 index 000000000000..f86dc872a79e --- /dev/null +++ b/tests/make/test_compile_locales.py @@ -0,0 +1,89 @@ +import subprocess +import sys +import tempfile +from pathlib import Path +from unittest import TestCase, mock +from unittest.mock import Mock, patch + +import pytest + +from scripts.compile_locales import compile_locales, process_po_file +from tests import override_env + + +@pytest.mark.needs_locales_compilation +class TestCompileLocales(TestCase): + def setUp(self): + self.home_dir = Path(tempfile.mkdtemp()) + self.locale_dir = (self.home_dir / 'locale').mkdir() + + @patch.dict(sys.modules, {'dennis': None}) + def test_dennis_not_installed(self): + """Test that the script raises when dennis is not installed""" + self.assertRaises(ImportError, compile_locales) + + @patch.dict(sys.modules, {'dennis': Mock()}) + @patch('scripts.compile_locales.ThreadPoolExecutor') + def test_process_po_file(self, mock_executor): + """Test that the script processes po files""" + # Create po files + django_po = self.home_dir / 'locale' / 'django.po' + django_po.touch() + djangojs_po = self.home_dir / 'locale' / 'djangojs.po' + djangojs_po.touch() + + # Setup ThreadPoolExecutor mock + mock_executor_instance = Mock() + mock_executor.return_value.__enter__.return_value = mock_executor_instance + + with override_env(HOME=self.home_dir.as_posix()): + compile_locales() + + # Get the actual arguments passed to map + actual_args = mock_executor_instance.map.call_args[0] + self.assertEqual(actual_args[0], process_po_file) + self.assertEqual(list(actual_args[1]), [django_po, djangojs_po]) + + +class TestProcessPoFile(TestCase): + def setUp(self): + self.pofile = Path(tempfile.mkdtemp()) / 'django.po' + + mock_subprocess = patch('scripts.compile_locales.subprocess.run') + self.mock_subprocess = mock_subprocess.start() + self.addCleanup(mock_subprocess.stop) + + def test_process_po_file(self): + process_po_file(self.pofile.as_posix()) + self.assertTrue(self.pofile.with_suffix('.mo').exists()) + + assert self.mock_subprocess.call_args_list == [ + mock.call( + ['dennis-cmd', 'lint', '--errorsonly', self.pofile.as_posix()], + capture_output=True, + check=False, + ), + mock.call( + [ + 'msgfmt', + '-o', + self.pofile.with_suffix('.mo'), + self.pofile.as_posix(), + ], + check=True, + ), + ] + + def test_process_po_file_retries(self): + self.mock_subprocess.side_effect = subprocess.CalledProcessError( + returncode=1, + cmd=['dennis-cmd', 'lint', '--errorsonly', self.pofile.as_posix()], + ) + + with self.assertRaises(subprocess.CalledProcessError): + process_po_file(self.pofile.as_posix()) + + self.assertTrue(self.pofile.with_suffix('.mo').exists()) + + # We expect 3 attempts to process the file + self.assertEqual(self.mock_subprocess.call_count, 3) diff --git a/tests/make/test_sync_host_files.py b/tests/make/test_sync_host_files.py new file mode 100644 index 000000000000..66e8dd5e188a --- /dev/null +++ b/tests/make/test_sync_host_files.py @@ -0,0 +1,36 @@ +import json +import tempfile +from pathlib import Path +from unittest import TestCase, mock + +from scripts.sync_host_files import sync_host_files +from tests import override_env + + +@mock.patch('scripts.sync_host_files.subprocess.run') +class TestSyncHostFiles(TestCase): + def test_sync_host_files(self, mock_subprocess): + sync_host_files() + + mock_subprocess.assert_has_calls( + [ + mock.call(['make', 'update_deps'], check=True), + # mock.call(['make', 'compile_locales'], check=True), + # mock.call(['make', 'update_assets'], check=True), + ] + ) + + def test_sync_host_files_production(self, mock_subprocess): + mock_build = Path(tempfile.mktemp()) + mock_build.write_text(json.dumps({'target': 'production'})) + + with override_env(BUILD_INFO=mock_build.as_posix()): + sync_host_files() + + mock_subprocess.assert_has_calls( + [ + mock.call(['make', 'update_deps'], check=True), + mock.call(['make', 'compile_locales'], check=True), + mock.call(['make', 'update_assets'], check=True), + ] + ) diff --git a/tests/make/test_update_assets.py b/tests/make/test_update_assets.py new file mode 100644 index 000000000000..8a73ab3fb190 --- /dev/null +++ b/tests/make/test_update_assets.py @@ -0,0 +1,76 @@ +import tempfile +from pathlib import Path +from unittest import TestCase, mock + +from scripts.update_assets import clean_static_dirs, update_assets +from tests import override_env + + +class TestUpdateAssets(TestCase): + def setUp(self): + self.mocks = {} + for name in ['clean_static_dirs', 'subprocess.run']: + patch = mock.patch(f'scripts.update_assets.{name}') + self.mocks[name] = patch.start() + self.addCleanup(patch.stop) + + def test_update_assets(self): + update_assets() + + assert self.mocks['clean_static_dirs'].call_count == 1 + + assert self.mocks['subprocess.run'].call_args_list == [ + mock.call( + ['python3', 'manage.py', 'compress_assets'], check=True, env=mock.ANY + ), + mock.call( + ['python3', 'manage.py', 'generate_jsi18n_files'], + check=True, + env=mock.ANY, + ), + mock.call( + ['python3', 'manage.py', 'collectstatic', '--noinput'], + check=True, + env=mock.ANY, + ), + ] + + for call in self.mocks['subprocess.run'].call_args_list: + assert ( + call.kwargs['env']['DJANGO_SETTINGS_MODULE'] + == 'olympia.lib.settings_base' + ) + + def test_update_assets_with_verbose(self): + update_assets(verbose=True) + + assert self.mocks['clean_static_dirs'].call_args_list == [ + mock.call(True), + ] + + +class TestCleanStaticDirs(TestCase): + def setUp(self): + self.home = Path(tempfile.mkdtemp()) + + def _run_clean_static_dirs(self, verbose=False): + with override_env(HOME=self.home.as_posix()): + clean_static_dirs(verbose=verbose) + + def test_creates_dirs(self): + self._run_clean_static_dirs() + + assert self.home.joinpath('static-build').exists() + assert self.home.joinpath('site-static').exists() + + def test_empties_dirs(self): + self.home.joinpath('static-build').mkdir() + (self.home / 'static-build' / 'test.txt').touch() + + self.home.joinpath('site-static').mkdir() + (self.home / 'site-static' / 'test.txt').touch() + + self._run_clean_static_dirs() + + assert not (self.home / 'static-build' / 'test.txt').exists() + assert not (self.home / 'site-static' / 'test.txt').exists()