Merge branch 'main' into dependabot/pip/jinja2-3.1.5

CHANGELOG.md

@@ -1,5 +1,37 @@
 # CHANGELOG
 
+## v0.1.5 (2025-01-11)
+
+### Ci
+
+* ci: Tests for prune behavior ([`31173a1`](https://github.com/UCSD-E4E/e4e-data-management/commit/31173a1ad25897de965548767d7cbbf42fb85f36))
+
+* ci: Fixes failing tests ([`3810a02`](https://github.com/UCSD-E4E/e4e-data-management/commit/3810a029971cf5e146765d57fed36eb9bed5d12a))
+
+### Documentation
+
+* docs: Documents fixture ([`a98cfea`](https://github.com/UCSD-E4E/e4e-data-management/commit/a98cfead653cc847af04ae90ee23371062e91347))
+
+### Fix
+
+* fix: Adds exception logging to main invocation ([`9f0aa7f`](https://github.com/UCSD-E4E/e4e-data-management/commit/9f0aa7f21ccc75bc7e031bc7443ac148ada70d98))
+
+### Style
+
+* style: Fixes long line ([`0d795f7`](https://github.com/UCSD-E4E/e4e-data-management/commit/0d795f73b470b4824cbe9cac06dd52eb879ba1a3))
+
+### Unknown
+
+* Merge pull request #57 from UCSD-E4E/44-add-exception-and-traceback-to-logs
+
+fix: Adds exception logging to main invocation ([`cb9c732`](https://github.com/UCSD-E4E/e4e-data-management/commit/cb9c732cd6e8426513615b36aad287493e21ebbc))
+
+* Merge branch 'main' into 44-add-exception-and-traceback-to-logs ([`de9e0d5`](https://github.com/UCSD-E4E/e4e-data-management/commit/de9e0d5bbacd41a9ef8fcd7cce0e8ba5cd3f942c))
+
+* Merge pull request #91 from UCSD-E4E/34-remove-dataset-after-pushing-to-server
+
+34 remove dataset after pushing to server ([`8a811be`](https://github.com/UCSD-E4E/e4e-data-management/commit/8a811bec27556f1dfba5044f3875cbecdc9b38e4))
+
 ## v0.1.4 (2024-11-04)
 
 ### Fix

e4e_data_management/__init__.py

@@ -1,3 +1,3 @@
 '''E4E Data Management Tools
 '''
-__version__ = '0.1.4'
+__version__ = '0.1.5'

e4e_data_management/cli.py

@@ -14,7 +14,7 @@
 from e4e_data_management import __version__
 from e4e_data_management.core import DataManager
 from e4e_data_management.metadata import Metadata
-
+from e4e_data_management.data import Dataset
 T = TypeVar('T')
 @dataclass
 class Parameter:
@@ -28,71 +28,90 @@ class Parameter:
     validator: Callable[[T], bool]
 
 
-class DataMangerCLI:
+class DataManagerCLI:
     """Data Manager Command Line Interface
     """
     def __init__(self):
         self.__configure_logging()
         self._log = logging.getLogger('e4edm.cli')
         self._log.debug('Invoking version %s from %s', __version__, __file__)
-        self.app = DataManager.load()
-        commands = [
-            'init_dataset',
-            'init_mission',
-            'status',
-            'list',
-            'config',
-            'activate',
-            'add',
-            'commit',
-            'duplicate',
-            'validate',
-            'push',
-            'zip',
-            'unzip',
-            'prune',
-            'ls',
-        ]
-        self.parameters = [
-            Parameter(
-            name='dataset_dir',
-            getter=lambda: getattr(self.app, 'dataset_dir'),
-            setter=lambda x: setattr(self.app, 'dataset_dir', x),
-            parser=Path,
-            formatter=Path.as_posix,
-            validator=Path.is_dir
-            ),
-            Parameter(
-            name='version',
-            getter=lambda: getattr(self.app, 'version'),
-            setter=None,
-            parser=int,
-            formatter=str,
-            validator=None
-            )
-        ]
-        self.parser = argparse.ArgumentParser()
-        subparsers = self.parser.add_subparsers()
-        parsers = {cmd:subparsers.add_parser(cmd) for cmd in commands}
-
-        self.__configure_init_dataset_parser(parsers['init_dataset'])
-        self.__configure_init_mission_parser(parsers['init_mission'])
-        self.__configure_status_parser(parsers['status'])
-        self.__configure_list_parser(parsers['list'])
-        self.__configure_add_parser(parsers['add'])
-        self.__configure_commit_parser(parsers['commit'])
-        self.__configure_duplicate_parser(parsers['duplicate'])
-        self.__configure_push_parser(parsers['push'])
-        self.__configure_prune_parser(parsers['prune'])
-        self.__configure_config_parser(parsers['config'])
-        self.__configure_activate_parser(parsers['activate'])
-        self.__configure_ls_parser(parsers['ls'])
-        # self.__configure_validate_parser(parsers['validate'])
-        # self.__configure_zip_parser(parsers['zip'])
-        # self.__configure_unzip_parser(parsers['unzip'])
-
-        self.parser.add_argument('--version', action='version', version=f'e4edm {__version__}')
-        self.parser.set_defaults(func=self.parser.print_help)
+        try:
+            self.app = DataManager.load()
+            commands = [
+                'init_dataset',
+                'init_mission',
+                'status',
+                'list',
+                'config',
+                'activate',
+                'add',
+                'commit',
+                'duplicate',
+                'validate',
+                'push',
+                'zip',
+                'unzip',
+                'prune',
+                'ls',
+            ]
+            self.parameters = [
+                Parameter(
+                name='dataset_dir',
+                getter=lambda: getattr(self.app, 'dataset_dir'),
+                setter=lambda x: setattr(self.app, 'dataset_dir', x),
+                parser=Path,
+                formatter=Path.as_posix,
+                validator=Path.is_dir
+                ),
+                Parameter(
+                name='version',
+                getter=lambda: getattr(self.app, 'version'),
+                setter=None,
+                parser=int,
+                formatter=str,
+                validator=None
+                )
+            ]
+            self.parser = argparse.ArgumentParser()
+            subparsers = self.parser.add_subparsers()
+            parsers = {cmd:subparsers.add_parser(cmd) for cmd in commands}
+
+            self.__configure_init_dataset_parser(parsers['init_dataset'])
+            self.__configure_init_mission_parser(parsers['init_mission'])
+            self.__configure_status_parser(parsers['status'])
+            self.__configure_list_parser(parsers['list'])
+            self.__configure_add_parser(parsers['add'])
+            self.__configure_commit_parser(parsers['commit'])
+            self.__configure_duplicate_parser(parsers['duplicate'])
+            self.__configure_push_parser(parsers['push'])
+            self.__configure_prune_parser(parsers['prune'])
+            self.__configure_config_parser(parsers['config'])
+            self.__configure_activate_parser(parsers['activate'])
+            self.__configure_ls_parser(parsers['ls'])
+            self.__configure_validate_parser(parsers['validate'])
+            # self.__configure_zip_parser(parsers['zip'])
+            # self.__configure_unzip_parser(parsers['unzip'])
+
+            self.parser.add_argument('--version', action='version', version=f'e4edm {__version__}')
+            self.parser.set_defaults(func=self.parser.print_help)
+        except Exception as exc:
+            self._log.exception('Exception during application load/configuration')
+            raise exc
+
+    def __configure_validate_parser(self, parser: argparse.ArgumentParser):
+        parser.add_argument('root_dir', nargs='?', default=None, type=Path)
+        parser.set_defaults(func=self.__external_validate)
+
+    def __external_validate(self, root_dir: Optional[Path]):
+        if root_dir is None:
+            dataset = self.app.active_dataset
+        else:
+            dataset = Dataset.load(path=root_dir)
+
+        if not dataset.validate():
+            print('Dataset validation failed')
+        else:
+            print('Dataset valid')
 
     def __configure_logging(self) -> None:
         log_dir = Path(DataManager.dirs.user_log_dir).resolve()
@@ -253,14 +272,18 @@ def prune_cmd(self):
     def main(self):
         """Main function
         """
-        self._log.info("Invoked with %s", ' '.join(sys.argv))
-        args = self.parser.parse_args()
-        arg_dict = vars(args)
+        try:
+            self._log.info("Invoked with %s", ' '.join(sys.argv))
+            args = self.parser.parse_args()
+            arg_dict = vars(args)
 
-        arg_fn = args.func
-        arg_dict.pop('func')
+            arg_fn = args.func
+            arg_dict.pop('func')
 
-        arg_fn(**arg_dict)
+            arg_fn(**arg_dict)
+        except Exception as exc:
+            self._log.exception('Exception during main execution')
+            raise exc
 
     def __configure_ls_parser(self, parser: argparse.ArgumentParser):
         parser.add_argument('path', type=Path, default=Path('.'))
@@ -398,7 +421,7 @@ def __configure_init_dataset_parser(self, parser: argparse.ArgumentParser):
 def main():
     """Main bootstrap
     """
-    DataMangerCLI().main()
+    DataManagerCLI().main()
 
 if __name__ == '__main__':
     main()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "e4e-data-management"
-version = "0.1.4"
+version = "0.1.5"
 description = "E4E Data Management Tool (Python)"
 authors = [
     "Nathan Hui <[email protected]>",

tests/conftest.py

@@ -130,7 +130,8 @@ def create_single_mission_data(single_mission: Tuple[Mock, DataManager, Path],
         test_data (Tuple[Path, int, int]): Test Data
 
     Returns:
-        Tuple[Tuple[DataManager, Path], Tuple[Path, int, int]]: test app, test data
+        Tuple[Tuple[DataManager, Path], Tuple[Path, int, int]]: ((mocked application, data manager,
+        root directory), (data path, number of files, file size))
     """
     _, app, _ = single_mission
     data_dir, _, _ = test_data

tests/test_purge.py

@@ -0,0 +1,30 @@
+'''Test purging files
+'''
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Tuple
+from unittest.mock import Mock
+
+from e4e_data_management.core import DataManager
+
+
+def test_purge(single_mission_data: Tuple[Tuple[Mock, DataManager, Path], Tuple[Path, int, int]],
+               test_readme: Path):
+    """Tests purging data
+
+    Args:
+        single_mission_data (Tuple[Tuple[Mock, DataManager, Path], Tuple[Path, int, int]]): Test
+        Data
+        test_readme (Path): Test Readme
+    """
+    app_fixture, _ = single_mission_data
+    _, app, root_dir = app_fixture
+
+    app.add([test_readme], readme=True)
+    app.commit(readme=True)
+
+    with TemporaryDirectory() as push_dir:
+        app.push(Path(push_dir))
+    assert len(list(root_dir.glob('2023.03.02.Test.San Diego'))) == 1
+    app.prune()
+    assert len(list(root_dir.glob('2023.03.02.Test.San Diego'))) == 0