diff --git a/CHANGELOG.md b/CHANGELOG.md index 323225e..3df9dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # CHANGELOG +## v0.1.5 (2025-01-11) + +### Ci + +* ci: Tests for prune behavior ([`31173a1`](https://github.com/UCSD-E4E/e4e-data-management/commit/31173a1ad25897de965548767d7cbbf42fb85f36)) + +* ci: Fixes failing tests ([`3810a02`](https://github.com/UCSD-E4E/e4e-data-management/commit/3810a029971cf5e146765d57fed36eb9bed5d12a)) + +### Documentation + +* docs: Documents fixture ([`a98cfea`](https://github.com/UCSD-E4E/e4e-data-management/commit/a98cfead653cc847af04ae90ee23371062e91347)) + +### Fix + +* fix: Adds exception logging to main invocation ([`9f0aa7f`](https://github.com/UCSD-E4E/e4e-data-management/commit/9f0aa7f21ccc75bc7e031bc7443ac148ada70d98)) + +### Style + +* style: Fixes long line ([`0d795f7`](https://github.com/UCSD-E4E/e4e-data-management/commit/0d795f73b470b4824cbe9cac06dd52eb879ba1a3)) + +### Unknown + +* Merge pull request #57 from UCSD-E4E/44-add-exception-and-traceback-to-logs + +fix: Adds exception logging to main invocation ([`cb9c732`](https://github.com/UCSD-E4E/e4e-data-management/commit/cb9c732cd6e8426513615b36aad287493e21ebbc)) + +* Merge branch 'main' into 44-add-exception-and-traceback-to-logs ([`de9e0d5`](https://github.com/UCSD-E4E/e4e-data-management/commit/de9e0d5bbacd41a9ef8fcd7cce0e8ba5cd3f942c)) + +* Merge pull request #91 from UCSD-E4E/34-remove-dataset-after-pushing-to-server + +34 remove dataset after pushing to server ([`8a811be`](https://github.com/UCSD-E4E/e4e-data-management/commit/8a811bec27556f1dfba5044f3875cbecdc9b38e4)) + ## v0.1.4 (2024-11-04) ### Fix diff --git a/e4e_data_management/__init__.py b/e4e_data_management/__init__.py index c3f4b1c..a3aac58 100644 --- a/e4e_data_management/__init__.py +++ b/e4e_data_management/__init__.py @@ -1,3 +1,3 @@ '''E4E Data Management Tools ''' -__version__ = '0.1.4' +__version__ = '0.1.5' diff --git a/e4e_data_management/cli.py b/e4e_data_management/cli.py index 3c34fbd..6d11c0a 100644 --- a/e4e_data_management/cli.py +++ b/e4e_data_management/cli.py @@ -14,7 +14,7 @@ from e4e_data_management import __version__ from e4e_data_management.core import DataManager from e4e_data_management.metadata import Metadata - +from e4e_data_management.data import Dataset T = TypeVar('T') @dataclass class Parameter: @@ -28,71 +28,90 @@ class Parameter: validator: Callable[[T], bool] -class DataMangerCLI: +class DataManagerCLI: """Data Manager Command Line Interface """ def __init__(self): self.__configure_logging() self._log = logging.getLogger('e4edm.cli') self._log.debug('Invoking version %s from %s', __version__, __file__) - self.app = DataManager.load() - commands = [ - 'init_dataset', - 'init_mission', - 'status', - 'list', - 'config', - 'activate', - 'add', - 'commit', - 'duplicate', - 'validate', - 'push', - 'zip', - 'unzip', - 'prune', - 'ls', - ] - self.parameters = [ - Parameter( - name='dataset_dir', - getter=lambda: getattr(self.app, 'dataset_dir'), - setter=lambda x: setattr(self.app, 'dataset_dir', x), - parser=Path, - formatter=Path.as_posix, - validator=Path.is_dir - ), - Parameter( - name='version', - getter=lambda: getattr(self.app, 'version'), - setter=None, - parser=int, - formatter=str, - validator=None - ) - ] - self.parser = argparse.ArgumentParser() - subparsers = self.parser.add_subparsers() - parsers = {cmd:subparsers.add_parser(cmd) for cmd in commands} - - self.__configure_init_dataset_parser(parsers['init_dataset']) - self.__configure_init_mission_parser(parsers['init_mission']) - self.__configure_status_parser(parsers['status']) - self.__configure_list_parser(parsers['list']) - self.__configure_add_parser(parsers['add']) - self.__configure_commit_parser(parsers['commit']) - self.__configure_duplicate_parser(parsers['duplicate']) - self.__configure_push_parser(parsers['push']) - self.__configure_prune_parser(parsers['prune']) - self.__configure_config_parser(parsers['config']) - self.__configure_activate_parser(parsers['activate']) - self.__configure_ls_parser(parsers['ls']) - # self.__configure_validate_parser(parsers['validate']) - # self.__configure_zip_parser(parsers['zip']) - # self.__configure_unzip_parser(parsers['unzip']) - - self.parser.add_argument('--version', action='version', version=f'e4edm {__version__}') - self.parser.set_defaults(func=self.parser.print_help) + try: + self.app = DataManager.load() + commands = [ + 'init_dataset', + 'init_mission', + 'status', + 'list', + 'config', + 'activate', + 'add', + 'commit', + 'duplicate', + 'validate', + 'push', + 'zip', + 'unzip', + 'prune', + 'ls', + ] + self.parameters = [ + Parameter( + name='dataset_dir', + getter=lambda: getattr(self.app, 'dataset_dir'), + setter=lambda x: setattr(self.app, 'dataset_dir', x), + parser=Path, + formatter=Path.as_posix, + validator=Path.is_dir + ), + Parameter( + name='version', + getter=lambda: getattr(self.app, 'version'), + setter=None, + parser=int, + formatter=str, + validator=None + ) + ] + self.parser = argparse.ArgumentParser() + subparsers = self.parser.add_subparsers() + parsers = {cmd:subparsers.add_parser(cmd) for cmd in commands} + + self.__configure_init_dataset_parser(parsers['init_dataset']) + self.__configure_init_mission_parser(parsers['init_mission']) + self.__configure_status_parser(parsers['status']) + self.__configure_list_parser(parsers['list']) + self.__configure_add_parser(parsers['add']) + self.__configure_commit_parser(parsers['commit']) + self.__configure_duplicate_parser(parsers['duplicate']) + self.__configure_push_parser(parsers['push']) + self.__configure_prune_parser(parsers['prune']) + self.__configure_config_parser(parsers['config']) + self.__configure_activate_parser(parsers['activate']) + self.__configure_ls_parser(parsers['ls']) + self.__configure_validate_parser(parsers['validate']) + # self.__configure_zip_parser(parsers['zip']) + # self.__configure_unzip_parser(parsers['unzip']) + + self.parser.add_argument('--version', action='version', version=f'e4edm {__version__}') + self.parser.set_defaults(func=self.parser.print_help) + except Exception as exc: + self._log.exception('Exception during application load/configuration') + raise exc + + def __configure_validate_parser(self, parser: argparse.ArgumentParser): + parser.add_argument('root_dir', nargs='?', default=None, type=Path) + parser.set_defaults(func=self.__external_validate) + + def __external_validate(self, root_dir: Optional[Path]): + if root_dir is None: + dataset = self.app.active_dataset + else: + dataset = Dataset.load(path=root_dir) + + if not dataset.validate(): + print('Dataset validation failed') + else: + print('Dataset valid') def __configure_logging(self) -> None: log_dir = Path(DataManager.dirs.user_log_dir).resolve() @@ -253,14 +272,18 @@ def prune_cmd(self): def main(self): """Main function """ - self._log.info("Invoked with %s", ' '.join(sys.argv)) - args = self.parser.parse_args() - arg_dict = vars(args) + try: + self._log.info("Invoked with %s", ' '.join(sys.argv)) + args = self.parser.parse_args() + arg_dict = vars(args) - arg_fn = args.func - arg_dict.pop('func') + arg_fn = args.func + arg_dict.pop('func') - arg_fn(**arg_dict) + arg_fn(**arg_dict) + except Exception as exc: + self._log.exception('Exception during main execution') + raise exc def __configure_ls_parser(self, parser: argparse.ArgumentParser): parser.add_argument('path', type=Path, default=Path('.')) @@ -398,7 +421,7 @@ def __configure_init_dataset_parser(self, parser: argparse.ArgumentParser): def main(): """Main bootstrap """ - DataMangerCLI().main() + DataManagerCLI().main() if __name__ == '__main__': main() diff --git a/pyproject.toml b/pyproject.toml index cc041ad..fe950da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "e4e-data-management" -version = "0.1.4" +version = "0.1.5" description = "E4E Data Management Tool (Python)" authors = [ "Nathan Hui ", diff --git a/tests/conftest.py b/tests/conftest.py index 38c14b3..8fb1435 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -130,7 +130,8 @@ def create_single_mission_data(single_mission: Tuple[Mock, DataManager, Path], test_data (Tuple[Path, int, int]): Test Data Returns: - Tuple[Tuple[DataManager, Path], Tuple[Path, int, int]]: test app, test data + Tuple[Tuple[DataManager, Path], Tuple[Path, int, int]]: ((mocked application, data manager, + root directory), (data path, number of files, file size)) """ _, app, _ = single_mission data_dir, _, _ = test_data diff --git a/tests/test_purge.py b/tests/test_purge.py new file mode 100644 index 0000000..8342e57 --- /dev/null +++ b/tests/test_purge.py @@ -0,0 +1,30 @@ +'''Test purging files +''' +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Tuple +from unittest.mock import Mock + +from e4e_data_management.core import DataManager + + +def test_purge(single_mission_data: Tuple[Tuple[Mock, DataManager, Path], Tuple[Path, int, int]], + test_readme: Path): + """Tests purging data + + Args: + single_mission_data (Tuple[Tuple[Mock, DataManager, Path], Tuple[Path, int, int]]): Test + Data + test_readme (Path): Test Readme + """ + app_fixture, _ = single_mission_data + _, app, root_dir = app_fixture + + app.add([test_readme], readme=True) + app.commit(readme=True) + + with TemporaryDirectory() as push_dir: + app.push(Path(push_dir)) + assert len(list(root_dir.glob('2023.03.02.Test.San Diego'))) == 1 + app.prune() + assert len(list(root_dir.glob('2023.03.02.Test.San Diego'))) == 0