From fab15363fcfd4365cf849e8dca8dbe136df9a57b Mon Sep 17 00:00:00 2001 From: Emre Sahin Date: Mon, 15 Jul 2024 13:01:04 +0300 Subject: [PATCH] Update Readme and Minor Fixes (#256) --- README.md | 62 ++-- book/src/ref/xvc-file-send.md | 2 +- book/src/ref/xvc-file-share.md | 4 +- ...xvc-pipeline-step-dependency-glob-items.md | 10 +- ...xvc-pipeline-step-dependency-line-items.md | 10 +- ...vc-pipeline-step-dependency-regex-items.md | 12 +- config/Cargo.toml | 6 +- config/src/lib.rs | 16 +- core/Cargo.toml | 12 +- core/src/aliases/mod.rs | 2 +- core/src/check_ignore/mod.rs | 2 +- core/src/error.rs | 4 +- core/src/root/mod.rs | 4 +- core/src/types/xvcroot.rs | 113 ++++--- ecs/Cargo.toml | 4 +- file/Cargo.toml | 16 +- file/src/hash/mod.rs | 4 +- file/src/lib.rs | 6 +- file/src/list/mod.rs | 1 - file/src/send/mod.rs | 6 +- file/src/share/mod.rs | 8 +- lib/Cargo.toml | 22 +- lib/src/api.rs | 6 +- lib/src/cli/mod.rs | 262 +++++++++----- lib/src/error.rs | 6 +- lib/src/git.rs | 7 +- lib/src/init/mod.rs | 8 +- lib/src/lib.rs | 4 +- lib/src/main.rs | 6 +- logging/Cargo.toml | 2 +- pipeline/Cargo.toml | 16 +- pipeline/src/lib.rs | 2 +- pipeline/src/pipeline/command.rs | 4 +- pipeline/src/pipeline/mod.rs | 78 ++--- storage/Cargo.toml | 16 +- storage/src/error.rs | 2 +- storage/src/lib.rs | 10 +- storage/src/storage/async_common.rs | 42 +-- storage/src/storage/digital_ocean.rs | 38 +-- storage/src/storage/gcs.rs | 320 +----------------- storage/src/storage/local.rs | 50 +-- storage/src/storage/minio.rs | 247 +------------- storage/src/storage/mod.rs | 40 +-- storage/src/storage/r2.rs | 34 +- storage/src/storage/rsync.rs | 20 +- storage/src/storage/s3.rs | 26 +- storage/src/storage/wasabi.rs | 18 +- test_helper/Cargo.toml | 4 +- walker/Cargo.toml | 6 +- workflow_tests/Cargo.toml | 22 +- workflow_tests/src/lib.rs | 4 +- workflow_tests/src/main.rs | 3 +- workflow_tests/tests/common/mod.rs | 2 +- workflow_tests/tests/test_init_1.rs | 2 +- workflow_tests/tests/test_pipeline_dag.rs | 1 + workflow_tests/tests/test_storage_new_s3.rs | 2 +- 56 files changed, 615 insertions(+), 1021 deletions(-) diff --git a/README.md b/README.md index bde251f1a..d67ce8722 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,14 @@ $ cargo install xvc [installed]: https://www.rust-lang.org/tools/install +If you want to use Xvc with Python console and Jupyter notebooks, you can also install it with `pip`: + +```shell +$ pip install xvc +``` + +Note that pip installation doesn't make `xvc` available as a shell command. Please see [xvc.py](https://github.com/iesahin/xvc.py) for usage details. + ## 🏃🏾 Quicktart Xvc seamlessly monitors your files and directories on top of Git. To commence, execute the following command within the repository: @@ -54,7 +62,7 @@ Include your data files and directories for tracking: $ xvc file track my-data/ --as symlink ``` -This command calculates content hashes for data (using BLAKE-3, by default) and logs them. The changes are committed to Git, and the files are copied to content-addressed directories within `.xvc/b3`. Additionally, read-only symbolic links to these directories are created. +This command calculates content hashes for data (using BLAKE-3, by default) and logs them. The changes are committed to Git, and the files are copied to content-addressed directories within `.xvc/b3`. Additionally, read-only symbolic links to these directories are created. You can specify different [recheck (checkout) methods](https://docs.xvc.dev/ref/xvc-file-recheck/) for files and directories, depending on your use case. If you need to track model files that change frequently, you can set recheck method `--as copy` (the default). @@ -66,13 +74,13 @@ $ xvc file track my-models/ --as copy Configure a cloud storage to share the files you added. ```shell -$ xvc storage new s3 --name my-remote --region us-east-1 --bucket-name my-xvc-remote +$ xvc storage new s3 --name my-storage --region us-east-1 --bucket-name my-xvc-remote ``` You can send the files to this storage. ```shell -$ xvc file send --to my-remote +$ xvc file send --to my-storage ``` When you (or someone else) want to access these files later, you can clone the Git repository and get the files from the @@ -83,7 +91,7 @@ $ git clone https://example.com/my-machine-learning-project Cloning into 'my-machine-learning-project'... $ cd my-machine-learning-project -$ xvc file bring my-data/ --from my-remote +$ xvc file bring my-data/ --from my-storage ``` @@ -103,15 +111,15 @@ The script uses the Faker library and this library must be available where you r $ xvc pipeline step new --step-name install-deps --command 'python3 -m pip install --quiet --user -r requirements.txt' ``` -We'll make this this step to depend on `requirements.txt` file, so when the file changes it will make the step run. +We'll make this this step to depend on `requirements.txt` file, so when the file changes it will make the step run. ```console $ xvc pipeline step dependency --step-name install-deps --file requirements.txt ``` -Xvc allows to create dependencies between pipeline steps. Dependent steps wait for dependencies to finish successfully. +Xvc allows to create dependencies between pipeline steps. Dependent steps wait for dependencies to finish successfully. -Now we create a step to run the script and make `install-deps` step a dependency of it. +Now we create a step to run the script and make `install-deps` step a dependency of it. ```console $ xvc pipeline step new --step-name generate-data --command 'python3 generate_data.py' @@ -124,40 +132,39 @@ After you define the pipeline, you can run it by: $ xvc pipeline run [DONE] install-deps (python3 -m pip install --quiet --user -r requirements.txt) [OUT] [generate-data] CSV file generated successfully. - + [DONE] generate-data (python3 generate_data.py) ``` -Xvc allows many kinds of dependnecies, like [files](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#file-dependencies), -[groups of files and directories defined by globs](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#glob-dependencies), -[regular expression searches in files](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#regex-dependencies), -[line ranges in files](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#line-dependencies), +Xvc allows many kinds of dependnecies, like [files](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#file-dependencies), +[groups of files and directories defined by globs](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#glob-dependencies), +[regular expression searches in files](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#regex-dependencies), +[line ranges in files](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#line-dependencies), [hyper-parameters defined in YAML, JSON or TOML files](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#hyper-parameter-dependencies) [HTTP URLs](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#url-dependencies), -[shell command outputs](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#generic-command-dependencies), -and [other steps](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#step-dependencies). +[shell command outputs](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#generic-command-dependencies), +and [other steps](https://docs.xvc.dev/ref/xvc-pipeline-step-dependency#step-dependencies). Suppose you're only interested in the IQ scores of those with _Dr._ in front of their names and how they differ from the rest in the dataset we created. Let's create a regex search dependency to the data file that will show all _doctors_ IQ scores. ```console -$ xvc pipeline step new --step-name dr-iq --command 'echo "${XVC_REGEX_ADDED_ITEMS}" >> dr-iq-scores.csv ' +$ xvc pipeline step new --step-name dr-iq --command 'echo "${XVC_ADDED_REGEX_ITEMS}" >> dr-iq-scores.csv ' $ xvc pipeline step dependency --step-name dr-iq --regex-items 'random_names_iq_scores.csv:/^Dr\..*' ``` -The first line specifies a command, when run writes `${XVC_REGEX_ADDED_ITEMS}` environment variable to `dr-iq-scores.csv` file. -The second line specifies the dependency which will also populate the `$[XVC_REGEX_ADDED_ITEMS]` environment variable in the command. +The first line specifies a command, when run writes `${XVC_ADDED_REGEX_ITEMS}` environment variable to `dr-iq-scores.csv` file. +The second line specifies the dependency which will also populate the `$[XVC_ADDED_REGEX_ITEMS]` environment variable in the command. -Some dependency types like [regex items], +Some dependency types like [regex items], [line items] and [glob items] inject environment variables in the commands they are a dependency. -For example, if you have two million files specified with a glob, but want to run a script only on the added files after the last run, you can use these environment variables. - +For example, if you have two million files specified with a glob, but want to run a script only on the added files after the last run, you can use these environment variables. When you run the pipeline again, a file named `dr-iq-scores.csv` will be created. Note that, as `requirements.txt` didn't change `install-deps` step and its dependent `generate-data` steps didn't run. ```console $ xvc pipeline run -[DONE] dr-iq (echo "${XVC_REGEX_ADDED_ITEMS}" >> dr-iq-scores.csv ) +[DONE] dr-iq (echo "${XVC_ADDED_REGEX_ITEMS}" >> dr-iq-scores.csv ) $ cat dr-iq-scores.csv Dr. Brian Shaffer,122 @@ -166,7 +173,7 @@ Dr. Mallory Payne MD,70 Dr. Sherry Leonard,93 Dr. Susan Swanson,81 -```` +``` We are using this feature to get lines starting with `Dr.` from the file and write them to another file. When the file changes, e.g. another record matching the dependency regex added to the `random_names_iq_scores.csv` file, it will also be added to `dr-iq-scores.csv` file. @@ -174,7 +181,7 @@ We are using this feature to get lines starting with `Dr.` from the file and wri $ zsh -cl 'echo "Dr. Albert Einstein,144" >> random_names_iq_scores.csv' $ xvc pipeline run -[DONE] dr-iq (echo "${XVC_REGEX_ADDED_ITEMS}" >> dr-iq-scores.csv ) +[DONE] dr-iq (echo "${XVC_ADDED_REGEX_ITEMS}" >> dr-iq-scores.csv ) $ cat dr-iq-scores.csv Dr. Brian Shaffer,122 @@ -285,7 +292,7 @@ $ cat my-pipeline.json "outputs": [] }, { - "command": "echo /"${XVC_REGEX_ADDED_ITEMS}/" >> dr-iq-scores.csv ", + "command": "echo /"${XVC_ADDED_REGEX_ITEMS}/" >> dr-iq-scores.csv ", "dependencies": [ { "RegexItems": { @@ -347,8 +354,7 @@ You can edit the file to change commands, add new dependencies, etc. and import $ xvc pipeline import --file my-pipeline.json --overwrite ``` -Lastly, if you noticed that the commands are long to type, there is an `xvc aliases` command that prints a set of aliases for commands. You can source the output in your `.zshrc` or `.bashrc`, and use the following commands instead, e.g., `xvc pipelines run` becomes `pvc run`. - +Lastly, if you noticed that the commands are long to type, there is an `xvc aliases` command that prints a set of aliases for commands. You can source the output in your `.zshrc` or `.bashrc`, and use the following commands instead, e.g., `xvc pipelines run` becomes `pvc run`. ```console $ xvc aliases @@ -450,7 +456,7 @@ And, biggest thanks to Rust designers, developers and contributors. Although I c - Star this repo. I feel very happy for every star and send my best wishes to you. That's a certain win to spend your two seconds for me. Thanks. - Use xvc. Tell me how it works for you, read the [documentation](https://docs.xvc.dev), [report bugs](https://github.com/iesahin/xvc/issues), [discuss features](https://github.com/iesahin/xvc/discussions). - Please note that, I don't accept large code PRs. Please open an issue to discuss your idea and write/modify a - reference page before sending a PR. I'm happy to discuss and help you to implement your idea. Also, it may require a copyright transfer to me, as there may be cases which I provide the code in other licenses. + reference page before sending a PR. I'm happy to discuss and help you to implement your idea. Also, it may require a copyright transfer to me, as there may be cases which I provide the code in other licenses. ## 📜 License @@ -461,7 +467,7 @@ Xvc is licensed under the [GNU GPL 3.0 License](https://github.com/iesahin/xvc/b I'm using Xvc daily and I'm happy with it. Tracking all my files with Git via arbitrary servers and cloud providers is something I always need. I'm happy to improve and maintain it as long as I use it. -Given that I'm working on this for the last two years for pure technical bliss, you can expect me to work on it more. +Given that I'm working on this for the last two years for pure technical bliss, you can expect me to work on it more. ## ⚠️ Disclaimer diff --git a/book/src/ref/xvc-file-send.md b/book/src/ref/xvc-file-send.md index 258803941..c64005bd1 100644 --- a/book/src/ref/xvc-file-send.md +++ b/book/src/ref/xvc-file-send.md @@ -12,7 +12,7 @@ Arguments: [TARGETS]... Targets to send/push/upload to storage Options: - -r, --remote Storage name or guid to send the files + -r, --storage Storage name or guid to send the files --force Force even if the files are already present in the storage -h, --help Print help diff --git a/book/src/ref/xvc-file-share.md b/book/src/ref/xvc-file-share.md index daa4567e1..cc59e03ed 100644 --- a/book/src/ref/xvc-file-share.md +++ b/book/src/ref/xvc-file-share.md @@ -6,13 +6,13 @@ $ xvc file share --help Share a file from S3 compatible storage for a limited time -Usage: xvc file share [OPTIONS] --remote +Usage: xvc file share [OPTIONS] --storage Arguments: File to send/push/upload to storage Options: - -r, --remote Storage name or guid to send the files + -r, --storage Storage name or guid to send the files -d, --duration Period to send the files to. You can use s, m, h, d, w suffixes [default: 24h] -h, --help Print help diff --git a/book/src/ref/xvc-pipeline-step-dependency-glob-items.md b/book/src/ref/xvc-pipeline-step-dependency-glob-items.md index 530b29799..9e2bc5c76 100644 --- a/book/src/ref/xvc-pipeline-step-dependency-glob-items.md +++ b/book/src/ref/xvc-pipeline-step-dependency-glob-items.md @@ -7,7 +7,7 @@ Unline glob dependency, glob items dependency keeps track of the individual file command run with the list of files from a glob and you want to track added and removed files, use this. Otherwise if your command for all the files in a glob and don't need to track which files have changed, use the glob dependency. -This one injects `${XVC_GLOB_ADDED_ITEMS}`, `${XVC_GLOB_REMOVED_ITEMS}`, `${XVC_GLOB_CHANGED_ITEMS}` and `${XVC_GLOB_ALL_ITEMS}` to the command +This one injects `${XVC_ADDED_GLOB_ITEMS}`, `${XVC_REMOVED_GLOB_ITEMS}`, `${XVC_CHANGED_GLOB_ITEMS}` and `${XVC_ALL_GLOB_ITEMS}` to the command environment. This command works only in Xvc repositories. @@ -41,7 +41,7 @@ $ tree Add a step to list the added files. ```console -$ xvc pipeline step new --step-name files-changed --command 'echo "### Added Files:\n${XVC_GLOB_ADDED_ITEMS}\n### Removed Files:\n${XVC_GLOB_REMOVED_ITEMS}\n### Changed Files:\n${XVC_GLOB_CHANGED_ITEMS}"' +$ xvc pipeline step new --step-name files-changed --command 'echo "### Added Files:\n${XVC_ADDED_GLOB_ITEMS}\n### Removed Files:\n${XVC_REMOVED_GLOB_ITEMS}\n### Changed Files:\n${XVC_CHANGED_GLOB_ITEMS}"' $ xvc pipeline step dependency --step-name files-changed --glob-items 'dir-*/*' @@ -63,7 +63,7 @@ dir-0002/file-0003.bin ### Changed Files: -[DONE] files-changed (echo "### Added Files:/n${XVC_GLOB_ADDED_ITEMS}/n### Removed Files:/n${XVC_GLOB_REMOVED_ITEMS}/n### Changed Files:/n${XVC_GLOB_CHANGED_ITEMS}") +[DONE] files-changed (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}") $ xvc pipeline run @@ -82,7 +82,7 @@ dir-0001/file-0001.bin ### Changed Files: -[DONE] files-changed (echo "### Added Files:/n${XVC_GLOB_ADDED_ITEMS}/n### Removed Files:/n${XVC_GLOB_REMOVED_ITEMS}/n### Changed Files:/n${XVC_GLOB_CHANGED_ITEMS}") +[DONE] files-changed (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}") ``` @@ -99,6 +99,6 @@ $ xvc pipeline run ### Changed Files: dir-0001/file-0002.bin -[DONE] files-changed (echo "### Added Files:/n${XVC_GLOB_ADDED_ITEMS}/n### Removed Files:/n${XVC_GLOB_REMOVED_ITEMS}/n### Changed Files:/n${XVC_GLOB_CHANGED_ITEMS}") +[DONE] files-changed (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}") ``` diff --git a/book/src/ref/xvc-pipeline-step-dependency-line-items.md b/book/src/ref/xvc-pipeline-step-dependency-line-items.md index dc851996a..587361088 100644 --- a/book/src/ref/xvc-pipeline-step-dependency-line-items.md +++ b/book/src/ref/xvc-pipeline-step-dependency-line-items.md @@ -5,8 +5,8 @@ You can make your steps to depend on lines of text files. The lines are defined When the text in those lines change, the step is invalidated. Unlike line dependencies, this dependency type keeps track of the lines in the -file. You can use `${XVC_LINE_ALL_ITEMS}`, `${XVC_LINE_ADDED_ITEMS}`, and -`${XVC_LINE_REMOVED_ITEMS}` environment variables in the command. Please be +file. You can use `${XVC_ALL_LINE_ITEMS}`, `${XVC_ADDED_LINE_ITEMS}`, and +`${XVC_REMOVED_LINE_ITEMS}` environment variables in the command. Please be aware that for large set of lines, this dependency can take up considerable space to keep track of all lines and if you don't need to keep track of changed lines, you can use `--lines` dependency. @@ -49,7 +49,7 @@ $ cat people.csv Let's a step to show the first 10 lines of the file: ```console -$ xvc pipeline step new --step-name print-top-10 --command 'echo "Added Lines:\n ${XVC_LINE_ADDED_ITEMS}\nRemoved Lines:\n${XVC_LINE_REMOVED_ITEMS}"' +$ xvc pipeline step new --step-name print-top-10 --command 'echo "Added Lines:\n ${XVC_ADDED_LINE_ITEMS}\nRemoved Lines:\n${XVC_REMOVED_LINE_ITEMS}"' ``` @@ -77,7 +77,7 @@ $ xvc pipeline run Removed Lines: -[DONE] print-top-10 (echo "Added Lines:/n ${XVC_LINE_ADDED_ITEMS}/nRemoved Lines:/n${XVC_LINE_REMOVED_ITEMS}") +[DONE] print-top-10 (echo "Added Lines:/n ${XVC_ADDED_LINE_ITEMS}/nRemoved Lines:/n${XVC_REMOVED_LINE_ITEMS}") `````` @@ -104,7 +104,7 @@ $ xvc pipeline run Removed Lines: "Hank", "M", 30, 71, 158 -[DONE] print-top-10 (echo "Added Lines:/n ${XVC_LINE_ADDED_ITEMS}/nRemoved Lines:/n${XVC_LINE_REMOVED_ITEMS}") +[DONE] print-top-10 (echo "Added Lines:/n ${XVC_ADDED_LINE_ITEMS}/nRemoved Lines:/n${XVC_REMOVED_LINE_ITEMS}") ``` diff --git a/book/src/ref/xvc-pipeline-step-dependency-regex-items.md b/book/src/ref/xvc-pipeline-step-dependency-regex-items.md index 96f4cf13a..2adeb9b50 100644 --- a/book/src/ref/xvc-pipeline-step-dependency-regex-items.md +++ b/book/src/ref/xvc-pipeline-step-dependency-regex-items.md @@ -4,7 +4,7 @@ You can specify a regular expression matched against the lines from a file as a the matched results changed. Unlike regex dependencies, regex item dependencies keep track of the matched items. You can access them with -`${XVC_REGEX_ALL_ITEMS}`, `${XVC_REGEX_ADDED_ITEMS}`, and `${XVC_REGEX_REMOVED_ITEMS}` environment variables. +`${XVC_ALL_REGEX_ITEMS}`, `${XVC_ADDED_REGEX_ITEMS}`, and `${XVC_REMOVED_REGEX_ITEMS}` environment variables. This command works only in Xvc repositories. @@ -44,8 +44,8 @@ $ cat people.csv Now, let's add steps to the pipeline to count males and females in the file: ```console -$ xvc pipeline step new --step-name new-males --command 'echo "New Males:\n ${XVC_REGEX_ADDED_ITEMS}"' -$ xvc pipeline step new --step-name new-females --command 'echo "New Females:\n ${XVC_REGEX_ADDED_ITEMS}"' +$ xvc pipeline step new --step-name new-males --command 'echo "New Males:\n ${XVC_ADDED_REGEX_ITEMS}"' +$ xvc pipeline step new --step-name new-females --command 'echo "New Females:\n ${XVC_ADDED_REGEX_ITEMS}"' $ xvc pipeline step dependency --step-name new-females --step new-males ``` @@ -77,7 +77,7 @@ $ xvc pipeline run "Omar", "M", 38, 70, 145 "Quin", "M", 29, 71, 176 -[DONE] new-males (echo "New Males:/n ${XVC_REGEX_ADDED_ITEMS}") +[DONE] new-males (echo "New Males:/n ${XVC_ADDED_REGEX_ITEMS}") [OUT] [new-females] New Females: "Elly", "F", 30, 66, 124 "Fran", "F", 33, 66, 115 @@ -87,7 +87,7 @@ $ xvc pipeline run "Page", "F", 31, 67, 135 "Ruth", "F", 28, 65, 131 -[DONE] new-females (echo "New Females:/n ${XVC_REGEX_ADDED_ITEMS}") +[DONE] new-females (echo "New Females:/n ${XVC_ADDED_REGEX_ITEMS}") `````` @@ -130,6 +130,6 @@ $ xvc pipeline run [OUT] [new-females] New Females: "Asude", "F", 12, 55, 110 -[DONE] new-females (echo "New Females:/n ${XVC_REGEX_ADDED_ITEMS}") +[DONE] new-females (echo "New Females:/n ${XVC_ADDED_REGEX_ITEMS}") ``` diff --git a/config/Cargo.toml b/config/Cargo.toml index 521db26c0..58eb69b32 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-config" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Xvc configuration management" authors = ["Emre Şahin "] @@ -20,8 +20,8 @@ debug = true [dependencies] -xvc-logging = { version = "0.6.7", path = "../logging" } -xvc-walker = { version = "0.6.7", path = "../walker" } +xvc-logging = { version = "0.6.8", path = "../logging" } +xvc-walker = { version = "0.6.8", path = "../walker" } ## Cli and config diff --git a/config/src/lib.rs b/config/src/lib.rs index 3518bd605..960023c29 100755 --- a/config/src/lib.rs +++ b/config/src/lib.rs @@ -155,7 +155,7 @@ pub struct XvcConfigMap { /// /// It's possible to ignore certain sources by supplying `None` to their values here. #[derive(Debug, Clone)] -pub struct XvcConfigInitParams { +pub struct XvcConfigParams { /// The default configuration for the project. /// It should contain all default values as a TOML document. /// Xvc produces this in [xvc_core::default_configuration]. @@ -171,10 +171,12 @@ pub struct XvcConfigInitParams { /// If `true`, it's read from [USER_CONFIG_DIRS]. pub include_user_config: bool, /// Where should we load the project's (public) configuration? - /// If `None`, it's ignored. + /// It's loaded in [XvcRootInner::new] + /// TODO: Add a option to ignore this pub project_config_path: Option, /// Where should we load the project's (private) configuration? - /// If `None`, it's ignored. + /// It's loaded in [XvcRootInner::new] + /// TODO: Add a option to ignore this pub local_config_path: Option, /// Should we include configuration from the environment. /// If `true`, look for all variables in the form @@ -196,14 +198,12 @@ pub struct XvcConfigInitParams { pub struct XvcConfig { /// Current directory. It can be set with xvc -C option pub current_dir: XvcConfigOption, - // /// The root if the command is happen to be run in XVC directory - // pub xvc_root: XvcConfigOption>, /// Configuration values for each level pub config_maps: Vec, /// The current configuration map, updated cascadingly pub the_config: HashMap, /// The init params used to create this config - pub init_params: XvcConfigInitParams, + pub init_params: XvcConfigParams, } impl fmt::Display for XvcConfig { @@ -225,7 +225,7 @@ impl XvcConfig { /// Loads the default configuration from `p`. /// /// The configuration must be a valid TOML document. - fn default_conf(p: &XvcConfigInitParams) -> Self { + fn default_conf(p: &XvcConfigParams) -> Self { let default_conf = p .default_configuration .parse::() @@ -482,7 +482,7 @@ impl XvcConfig { /// Loads all config files /// Overrides all options with the given key=value style options in the /// command line - pub fn new(p: XvcConfigInitParams) -> Result { + pub fn new(p: XvcConfigParams) -> Result { let mut config = XvcConfig::default_conf(&p); config.current_dir = XvcConfigOption { diff --git a/core/Cargo.toml b/core/Cargo.toml index c1c4b6918..86c8d55ec 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-core" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Xvc core for common elements for all commands" authors = ["Emre Şahin "] @@ -19,10 +19,10 @@ crate-type = ["rlib"] debug = true [dependencies] -xvc-config = { version = "0.6.7", path = "../config" } -xvc-logging = { version = "0.6.7", path = "../logging" } -xvc-ecs = { version = "0.6.7", path = "../ecs" } -xvc-walker = { version = "0.6.7", path = "../walker" } +xvc-config = { version = "0.6.8", path = "../config" } +xvc-logging = { version = "0.6.8", path = "../logging" } +xvc-ecs = { version = "0.6.8", path = "../ecs" } +xvc-walker = { version = "0.6.8", path = "../walker" } ## Cli and config clap = { version = "^4.4", features = ["derive"] } @@ -82,6 +82,6 @@ derive_more = "^0.99" itertools = "^0.12" [dev-dependencies] -xvc-test-helper = { version = "0.6.7", path = "../test_helper/" } +xvc-test-helper = { version = "0.6.8", path = "../test_helper/" } proptest = "^1.4" test-case = "^3.3" diff --git a/core/src/aliases/mod.rs b/core/src/aliases/mod.rs index dbfcc91fc..3e550cc36 100644 --- a/core/src/aliases/mod.rs +++ b/core/src/aliases/mod.rs @@ -4,7 +4,7 @@ use clap::Parser; use xvc_logging::{output, XvcOutputSender}; -#[derive(Debug, Parser)] +#[derive(Debug, Parser, Clone)] #[command(name = "aliases")] /// Print aliases in the common format to be added to `.zsh_aliases`, `.bash_aliases` or /// `.profile`. diff --git a/core/src/check_ignore/mod.rs b/core/src/check_ignore/mod.rs index d4cd002b1..686af5eec 100644 --- a/core/src/check_ignore/mod.rs +++ b/core/src/check_ignore/mod.rs @@ -80,7 +80,7 @@ pub fn cmd_check_ignore( ignore_filename: Some(opts.ignore_filename.clone()), include_dirs: true, }; - let initial_rules = IgnoreRules::try_from_patterns(xvc_root, COMMON_IGNORE_PATTERNS)?; + let initial_rules = IgnoreRules::try_from_patterns(&xvc_root, COMMON_IGNORE_PATTERNS)?; let ignore_rules = build_ignore_rules( initial_rules, current_dir, diff --git a/core/src/error.rs b/core/src/error.rs index 53048c5d8..f821dc3ea 100644 --- a/core/src/error.rs +++ b/core/src/error.rs @@ -45,9 +45,9 @@ pub enum Error { #[error("File System Walk Error: {error}")] FSWalkerError { error: String }, - #[error("Cannot find XVC Root: {path}")] + #[error("Cannot find Xvc Root: {path}")] CannotFindXvcRoot { path: PathBuf }, - #[error("Cannot nest XVC repositories: {path}")] + #[error("Cannot nest Xvc repositories: {path}")] CannotNestXvcRepositories { path: PathBuf }, #[error("Regex Error: {source}")] RegexError { diff --git a/core/src/root/mod.rs b/core/src/root/mod.rs index 6b24d58a5..db8ead2bb 100644 --- a/core/src/root/mod.rs +++ b/core/src/root/mod.rs @@ -6,9 +6,9 @@ use clap::Parser; use relative_path::RelativePath; use xvc_logging::{output, watch, XvcOutputSender}; -#[derive(Debug, Parser)] +#[derive(Debug, Parser, Clone)] #[command(name = "root")] -/// Get the XVC root directory for the current project +/// Get the Xvc root directory for the current project pub struct RootCLI { #[arg(long)] /// Show absolute path instead of relative diff --git a/core/src/types/xvcroot.rs b/core/src/types/xvcroot.rs index dfde9716d..1f7b5124a 100644 --- a/core/src/types/xvcroot.rs +++ b/core/src/types/xvcroot.rs @@ -10,10 +10,10 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use xvc_ecs::ecs::timestamp; use xvc_ecs::{XvcEntity, XvcEntityGenerator}; -use xvc_logging::{debug, trace}; +use xvc_logging::watch; use xvc_walker::AbsolutePath; -use xvc_config::{XvcConfig, XvcConfigInitParams}; +use xvc_config::{XvcConfig, XvcConfigParams}; use crate::error::{Error, Result}; use crate::GITIGNORE_INITIAL_CONTENT; @@ -69,45 +69,23 @@ impl Deref for XvcRootInner { /// The path is not required to be the root of the repository. /// This function searches for the root of the repository using /// [XvcRoot::find_root] and uses it as the root. -pub fn load_xvc_root(path: &Path, config_opts: XvcConfigInitParams) -> Result { +pub fn load_xvc_root(config_opts: XvcConfigParams) -> Result { + let path = config_opts.current_dir.as_ref(); + match XvcRootInner::find_root(path) { Ok(absolute_path) => { - let xvc_dir = absolute_path.join(XvcRootInner::XVC_DIR); - let local_config_path = xvc_dir.join(XvcRootInner::LOCAL_CONFIG_PATH); - let project_config_path = xvc_dir.join(XvcRootInner::PROJECT_CONFIG_PATH); - let config_opts = XvcConfigInitParams { - project_config_path: Some(project_config_path.clone()), - local_config_path: Some(local_config_path.clone()), - default_configuration: config_opts.default_configuration, - current_dir: config_opts.current_dir, - include_system_config: config_opts.include_system_config, - include_user_config: config_opts.include_user_config, - include_environment_config: config_opts.include_environment_config, - command_line_config: config_opts.command_line_config, - }; - let config = XvcConfig::new(config_opts)?; - let entity_generator = - xvc_ecs::load_generator(&xvc_dir.join(XvcRootInner::ENTITY_GENERATOR_PATH))?; - - let store_dir = xvc_dir.join(XvcRootInner::STORE_DIR); - let xvc_root = Arc::new(XvcRootInner { - xvc_dir, - store_dir, - local_config_path, - project_config_path, - absolute_path, - config, - entity_generator, - }); - Ok(xvc_root) + Ok(Arc::new(XvcRootInner::new(absolute_path, config_opts)?)) + } + Err(e) => { + watch!(&e); + Err(e) } - Err(e) => Err(e), } } /// Creates a new .xvc dir in `path` and initializes a directory. /// *Warning:* This should only be used in `xvc init`, not in other commands. -pub fn init_xvc_root(path: &Path, config_opts: XvcConfigInitParams) -> Result { +pub fn init_xvc_root(path: &Path, config_opts: XvcConfigParams) -> Result { match XvcRootInner::find_root(path) { Ok(abs_path) => Err(Error::CannotNestXvcRepositories { path: abs_path.to_path_buf(), @@ -126,7 +104,7 @@ pub fn init_xvc_root(path: &Path, config_opts: XvcConfigInitParams) -> Result Result Result Result { + let xvc_dir = absolute_path.join(XvcRootInner::XVC_DIR); + let local_config_path = xvc_dir.join(XvcRootInner::LOCAL_CONFIG_PATH); + let project_config_path = xvc_dir.join(XvcRootInner::PROJECT_CONFIG_PATH); + let config_opts = XvcConfigParams { + project_config_path: Some(project_config_path.clone()), + local_config_path: Some(local_config_path.clone()), + default_configuration: config_opts.default_configuration, + current_dir: config_opts.current_dir, + include_system_config: config_opts.include_system_config, + include_user_config: config_opts.include_user_config, + include_environment_config: config_opts.include_environment_config, + command_line_config: config_opts.command_line_config, + }; + let config = XvcConfig::new(config_opts)?; + let entity_generator = + xvc_ecs::load_generator(&xvc_dir.join(XvcRootInner::ENTITY_GENERATOR_PATH))?; + + let store_dir = xvc_dir.join(XvcRootInner::STORE_DIR); + Ok(Self { + xvc_dir, + store_dir, + local_config_path, + project_config_path, + absolute_path, + config, + entity_generator, + }) + } + /// Join `path` to the repository root and return the absolute path of the /// given path. /// @@ -244,26 +254,24 @@ impl XvcRootInner { /// Finds the root of the xvc repository by looking for the .xvc directory /// in parents of a given path. pub fn find_root(path: &Path) -> Result { - trace!("{:?}", path); - let mut pb = PathBuf::from(path) + let abs_path = PathBuf::from(path) .canonicalize() .expect("Cannot canonicalize the path. Possible symlink loop."); - loop { - if pb.join(XVC_DIR).is_dir() { - debug!("XVC DIR: {:?}", pb); - return Ok(pb.into()); - } else if pb.parent().is_none() { - return Err(Error::CannotFindXvcRoot { path: path.into() }); - } else { - pb.pop(); + + for parent in abs_path.ancestors() { + let xvc_candidate = parent.join(XVC_DIR); + watch!(xvc_candidate); + if parent.join(XVC_DIR).is_dir() { + watch!(parent); + return Ok(parent.into()); } } + Err(Error::CannotFindXvcRoot { path: path.into() }) } -} -impl Drop for XvcRootInner { - /// Saves the entity_generator before dropping - fn drop(&mut self) { + + /// Record the entity generator to the disk + pub fn record(&self) { match self.entity_generator.save(&self.entity_generator_path()) { Ok(_) => (), Err(e) => { @@ -272,3 +280,10 @@ impl Drop for XvcRootInner { } } } + +impl Drop for XvcRootInner { + /// Saves the entity_generator before dropping + fn drop(&mut self) { + self.record() + } +} diff --git a/ecs/Cargo.toml b/ecs/Cargo.toml index 4e43525a9..835e749af 100644 --- a/ecs/Cargo.toml +++ b/ecs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-ecs" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Entity-Component System for Xvc" authors = ["Emre Şahin "] @@ -19,7 +19,7 @@ crate-type = ["rlib"] debug = true [dependencies] -xvc-logging = { version = "0.6.7", path = "../logging" } +xvc-logging = { version = "0.6.8", path = "../logging" } ## Serialization serde = { version = "^1.0", features = ["derive"] } diff --git a/file/Cargo.toml b/file/Cargo.toml index 22d3ee19f..98e5df38d 100644 --- a/file/Cargo.toml +++ b/file/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-file" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "File tracking, versioning, upload and download functions for Xvc" authors = ["Emre Şahin "] @@ -24,12 +24,12 @@ bench = true debug = true [dependencies] -xvc-logging = { version = "0.6.7", path = "../logging" } -xvc-config = { version = "0.6.7", path = "../config" } -xvc-core = { version = "0.6.7", path = "../core" } -xvc-ecs = { version = "0.6.7", path = "../ecs" } -xvc-walker = { version = "0.6.7", path = "../walker" } -xvc-storage = { version = "0.6.7", path = "../storage" } +xvc-logging = { version = "0.6.8", path = "../logging" } +xvc-config = { version = "0.6.8", path = "../config" } +xvc-core = { version = "0.6.8", path = "../core" } +xvc-ecs = { version = "0.6.8", path = "../ecs" } +xvc-walker = { version = "0.6.8", path = "../walker" } +xvc-storage = { version = "0.6.8", path = "../storage" } ## Cli and config @@ -94,5 +94,5 @@ parse-size = "^1.0" [dev-dependencies] -xvc-test-helper = { version = "0.6.7", path = "../test_helper/" } +xvc-test-helper = { version = "0.6.8", path = "../test_helper/" } shellfn = "^0.1" diff --git a/file/src/hash/mod.rs b/file/src/hash/mod.rs index 05b74887f..b7db288f3 100644 --- a/file/src/hash/mod.rs +++ b/file/src/hash/mod.rs @@ -7,7 +7,7 @@ use clap::Parser; use crossbeam_channel::unbounded; use log::warn; use std::{env, path::PathBuf}; -use xvc_config::{FromConfigKey, UpdateFromXvcConfig, XvcConfig, XvcConfigInitParams}; +use xvc_config::{FromConfigKey, UpdateFromXvcConfig, XvcConfig, XvcConfigParams}; use xvc_core::ContentDigest; use xvc_core::{ util::file::{path_metadata_channel, pipe_filter_path_errors}, @@ -63,7 +63,7 @@ pub fn cmd_hash( ) -> Result<()> { let conf = match xvc_root { Some(xvc_root) => xvc_root.config().clone(), - None => XvcConfig::new(XvcConfigInitParams { + None => XvcConfig::new(XvcConfigParams { default_configuration: xvc_core::default_project_config(false), current_dir: AbsolutePath::from(env::current_dir()?), include_system_config: true, diff --git a/file/src/lib.rs b/file/src/lib.rs index 16dc771dd..e95803b98 100755 --- a/file/src/lib.rs +++ b/file/src/lib.rs @@ -47,7 +47,7 @@ use std::io; use std::io::Write; use std::path::Path; use std::path::PathBuf; -use xvc_config::XvcConfigInitParams; +use xvc_config::XvcConfigParams; use xvc_config::XvcVerbosity; use xvc_core::default_project_config; use xvc_core::types::xvcroot::load_xvc_root; @@ -265,7 +265,7 @@ pub fn dispatch(cli_opts: XvcFileCLI) -> Result<()> { AbsolutePath::from(std::env::current_dir()?.join(dir).canonicalize()?) }; // try to create root - let xvc_config_params = XvcConfigInitParams { + let xvc_config_params = XvcConfigParams { current_dir, include_system_config: !cli_opts.no_system_config, include_user_config: !cli_opts.no_user_config, @@ -276,7 +276,7 @@ pub fn dispatch(cli_opts: XvcFileCLI) -> Result<()> { default_configuration: default_project_config(true), }; - let xvc_root = match load_xvc_root(Path::new(&cli_opts.workdir), xvc_config_params) { + let xvc_root = match load_xvc_root(xvc_config_params) { Ok(r) => Some(r), Err(e) => { e.info(); diff --git a/file/src/list/mod.rs b/file/src/list/mod.rs index b577ddae6..e4f45aa2c 100644 --- a/file/src/list/mod.rs +++ b/file/src/list/mod.rs @@ -127,7 +127,6 @@ struct ListRow { } impl ListRow { - /// fn new(path_prefix: &Path, path_match: PathMatch) -> Result { let actual_file_type = String::from(if let Some(actual_metadata) = path_match.actual_metadata { diff --git a/file/src/send/mod.rs b/file/src/send/mod.rs index 0d2bde9b1..6c71246a6 100644 --- a/file/src/send/mod.rs +++ b/file/src/send/mod.rs @@ -7,8 +7,6 @@ use crate::Result; use clap::Parser; -use humantime; - use xvc_core::{ContentDigest, XvcCachePath, XvcFileType, XvcMetadata, XvcRoot}; use xvc_ecs::{HStore, XvcStore}; use xvc_logging::{error, watch, XvcOutputSender}; @@ -25,7 +23,7 @@ use xvc_storage::{storage::get_storage_record, StorageIdentifier, XvcStorageOper pub struct SendCLI { /// Storage name or guid to send the files #[arg(long, short, alias = "to")] - remote: StorageIdentifier, + storage: StorageIdentifier, /// Force even if the files are already present in the storage #[arg(long)] force: bool, @@ -36,7 +34,7 @@ pub struct SendCLI { /// Send a targets in `opts.targets` in `xvc_root` to `opt.remote` pub fn cmd_send(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: SendCLI) -> Result<()> { - let remote = get_storage_record(output_snd, xvc_root, &opts.remote)?; + let remote = get_storage_record(output_snd, xvc_root, &opts.storage)?; watch!(remote); let current_dir = xvc_root.config().current_dir()?; let targets = load_targets_from_store(xvc_root, current_dir, &opts.targets)?; diff --git a/file/src/share/mod.rs b/file/src/share/mod.rs index c16c82e51..6fe3d30e0 100644 --- a/file/src/share/mod.rs +++ b/file/src/share/mod.rs @@ -17,7 +17,7 @@ use xvc_storage::{storage::get_storage_record, StorageIdentifier, XvcStorageOper pub struct ShareCLI { /// Storage name or guid to send the files #[arg(long, short, alias = "from")] - remote: StorageIdentifier, + storage: StorageIdentifier, /// Period to send the files to. You can use s, m, h, d, w suffixes. #[arg(long, short, default_value = "24h")] duration: String, @@ -28,8 +28,8 @@ pub struct ShareCLI { pub fn cmd_share(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: ShareCLI) -> Result<()> { // TODO: TIDY UP these implementation to reuse code in other places - let remote = get_storage_record(output_snd, xvc_root, &opts.remote)?; - watch!(remote); + let storage = get_storage_record(output_snd, xvc_root, &opts.storage)?; + watch!(storage); let current_dir = xvc_root.config().current_dir()?; let targets = load_targets_from_store(xvc_root, current_dir, &Some(vec![opts.target]))?; watch!(targets); @@ -66,6 +66,6 @@ pub fn cmd_share(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: ShareCL watch!(duration); - remote.share(output_snd, xvc_root, &cache_path, duration)?; + storage.share(output_snd, xvc_root, &cache_path, duration)?; Ok(()) } diff --git a/lib/Cargo.toml b/lib/Cargo.toml index bca10752e..3a306ad5c 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "xvc" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "An MLOps tool to manage data files and pipelines on top of Git" authors = ["Emre Şahin "] -license = "GPL-2.0" +license = "GPL-3.0" homepage = "https://xvc.dev" repository = "https://github.com/iesahin/xvc" categories = ["command-line-utilities", "development-tools"] @@ -23,14 +23,14 @@ path = "src/main.rs" debug = true [dependencies] -xvc-config = { version = "0.6.7", path = "../config" } -xvc-core = { version = "0.6.7", path = "../core" } -xvc-logging = { version = "0.6.7", path = "../logging" } -xvc-ecs = { version = "0.6.7", path = "../ecs" } -xvc-file = { version = "0.6.7", path = "../file" } -xvc-pipeline = { version = "0.6.7", path = "../pipeline" } -xvc-walker = { version = "0.6.7", path = "../walker" } -xvc-storage = { version = "0.6.7", path = "../storage" } +xvc-config = { version = "0.6.8", path = "../config" } +xvc-core = { version = "0.6.8", path = "../core" } +xvc-logging = { version = "0.6.8", path = "../logging" } +xvc-ecs = { version = "0.6.8", path = "../ecs" } +xvc-file = { version = "0.6.8", path = "../file" } +xvc-pipeline = { version = "0.6.8", path = "../pipeline" } +xvc-walker = { version = "0.6.8", path = "../walker" } +xvc-storage = { version = "0.6.8", path = "../storage" } ## Cli and config @@ -113,4 +113,4 @@ jwalk = "^0.8" proptest = "^1.4" shellfn = "^0.1" test-case = "^3.3" -xvc-test-helper = { version = "0.6.7", path = "../test_helper/" } +xvc-test-helper = { version = "0.6.8", path = "../test_helper/" } diff --git a/lib/src/api.rs b/lib/src/api.rs index 38df23975..388bc4dca 100644 --- a/lib/src/api.rs +++ b/lib/src/api.rs @@ -19,11 +19,15 @@ pub use xvc_logging::warn; pub use xvc_logging::watch; pub use xvc_config::XvcConfig; -pub use xvc_config::XvcConfigInitParams; +pub use xvc_config::XvcConfigParams; pub use xvc_config::XvcConfigOptionSource; pub use xvc_core::AbsolutePath; +pub use xvc_core::XvcRoot; +/// Commands usually receive an optional xvc_root object for the repository +pub type XvcRootOpt = Option; + pub use xvc_file::BringCLI as XvcFileBringCLI; pub use xvc_file::CarryInCLI as XvcFileCarryInCLI; pub use xvc_file::CopyCLI as XvcFileCopyCLI; diff --git a/lib/src/cli/mod.rs b/lib/src/cli/mod.rs index 80c15744a..b019935e2 100644 --- a/lib/src/cli/mod.rs +++ b/lib/src/cli/mod.rs @@ -1,12 +1,14 @@ -//! Main CLI interface for XVC +//! Main CLI interface for Xvc use std::env::ArgsOs; use std::ffi::OsString; use std::path::PathBuf; +use std::str::FromStr; use crate::git_checkout_ref; use crate::handle_git_automation; use crate::init; +use crate::XvcRootOpt; use clap::Parser; use crossbeam::thread; @@ -14,10 +16,11 @@ use crossbeam_channel::bounded; use log::LevelFilter; use std::io; use xvc_core::types::xvcroot::load_xvc_root; +use xvc_core::types::xvcroot::XvcRootInner; +use xvc_core::XvcRoot; use xvc_logging::{debug, error, uwr, XvcOutputLine}; -use std::path::Path; -use xvc_config::{XvcConfigInitParams, XvcVerbosity}; +use xvc_config::{XvcConfigParams, XvcVerbosity}; use xvc_core::aliases; use xvc_core::check_ignore; use xvc_core::default_project_config; @@ -124,6 +127,17 @@ impl XvcCLI { }) } + /// Parse the given elements with [clap::Parser::parse_from] and merge them to set + /// [XvcCLI::command_string]. + pub fn from_string_slice(args: &[String]) -> Result { + let command_string = args.join(" "); + let parsed = Self::parse_from(args); + Ok(Self { + command_string, + ..parsed + }) + } + /// Parse the command line from the result of [`std::env::args_os`]. /// This updates [XvcCLI::command_string] with the command line. pub fn from_args_os(args_os: ArgsOs) -> Result { @@ -154,8 +168,24 @@ impl XvcCLI { } } +// Implement FromStr for XvcCLI + +impl FromStr for XvcCLI { + type Err = Error; + + fn from_str(s: &str) -> Result { + let command_string = s.to_owned(); + let args: Vec = s.split(' ').map(|a| a.trim().to_owned()).collect(); + let parsed = Self::parse_from(args); + Ok(Self { + command_string, + ..parsed + }) + } +} + /// Xvc subcommands -#[derive(Debug, Parser)] +#[derive(Debug, Parser, Clone)] #[command(rename_all = "kebab-case")] pub enum XvcSubCommand { /// File and directory management commands @@ -175,73 +205,28 @@ pub enum XvcSubCommand { } /// Runs the supplied xvc command. -pub fn run(args: &[&str]) -> Result<()> { +pub fn run(args: &[&str]) -> Result { let cli_options = cli::XvcCLI::from_str_slice(args)?; dispatch(cli_options) } -/// Dispatch commands to respective functions in the API -/// -/// It sets output verbosity with [XvcCLI::verbosity]. -/// Determines configuration sources by filling [XvcConfigInitParams]. -/// Tries to create an XvcRoot to determine whether we're inside one. -/// Creates two threads: One for running the API function, one for getting strings from output -/// channel. -/// -/// A corresponding function to reuse the same [XvcRoot] object is [test_dispatch]. -/// It doesn't recreate the whole configuration and this prevents errors regarding multiple -/// initializations. -pub fn dispatch(cli_opts: cli::XvcCLI) -> Result<()> { - let verbosity = if cli_opts.quiet { - XvcVerbosity::Quiet - } else { - match cli_opts.verbosity { - 0 => XvcVerbosity::Default, - 1 => XvcVerbosity::Warn, - 2 => XvcVerbosity::Info, - 3 => XvcVerbosity::Debug, - _ => XvcVerbosity::Trace, - } - }; - - let term_log_level = match verbosity { - XvcVerbosity::Quiet => LevelFilter::Off, - XvcVerbosity::Default => LevelFilter::Error, - XvcVerbosity::Warn => LevelFilter::Warn, - XvcVerbosity::Info => LevelFilter::Info, - XvcVerbosity::Debug => LevelFilter::Debug, - XvcVerbosity::Trace => LevelFilter::Trace, - }; - - setup_logging( - Some(term_log_level), - if cli_opts.debug { - Some(LevelFilter::Trace) - } else { - None - }, +pub fn dispatch_with_root(cli_opts: cli::XvcCLI, xvc_root_opt: XvcRootOpt) -> Result { + + // XvcRoot should be kept per repository and shouldn't change directory across runs + assert!( + xvc_root_opt.as_ref().is_none() + || xvc_root_opt + .as_ref() + .map( + |xvc_root| XvcRootInner::find_root(&cli_opts.workdir).unwrap() + == *xvc_root.absolute_path() + ) + .unwrap() ); - let xvc_config_params = XvcConfigInitParams { - current_dir: AbsolutePath::from(&cli_opts.workdir), - include_system_config: !cli_opts.no_system_config, - include_user_config: !cli_opts.no_user_config, - project_config_path: None, - local_config_path: None, - include_environment_config: !cli_opts.no_env_config, - command_line_config: Some(cli_opts.consolidate_config_options()), - default_configuration: default_project_config(true), - }; - - let xvc_root_opt = match load_xvc_root(Path::new(&cli_opts.workdir), xvc_config_params) { - Ok(r) => Some(r), - Err(e) => { - e.debug(); - None - } - }; + let term_log_level = get_term_log_level(get_verbosity(&cli_opts)); - thread::scope(move |s| { + let xvc_root_opt = thread::scope(move |s| { let (output_snd, output_rec) = bounded::>(CHANNEL_BOUND); let output_snd_clone = output_snd.clone(); @@ -316,76 +301,84 @@ pub fn dispatch(cli_opts: cli::XvcCLI) -> Result<()> { ); } } - let command_thread = s.spawn(move |_| -> Result<()> { - match cli_opts.command { + let xvc_root_opt_res = s.spawn(move |_| -> Result { + let xvc_root_opt = match cli_opts.command { XvcSubCommand::Init(opts) => { - let use_git = !opts.no_git; let xvc_root = init::run(xvc_root_opt.as_ref(), opts)?; - if use_git { - handle_git_automation( - &output_snd, - xvc_root, - cli_opts.to_branch.as_deref(), - &cli_opts.command_string, - )?; - } - Result::Ok(()) + Some(xvc_root) } - XvcSubCommand::Aliases(opts) => Ok(aliases::run(&output_snd, opts)?), + XvcSubCommand::Aliases(opts) => { + aliases::run(&output_snd, opts)?; + xvc_root_opt + }, // following commands can only be run inside a repository - XvcSubCommand::Root(opts) => Ok(root::run( + XvcSubCommand::Root(opts) => { root::run( &output_snd, xvc_root_opt .as_ref() .ok_or_else(|| Error::RequiresXvcRepository)?, opts, - )?), + )?; + xvc_root_opt + }, + XvcSubCommand::File(opts) => { - Ok(file::run(&output_snd, xvc_root_opt.as_ref(), opts)?) + file::run(&output_snd, xvc_root_opt.as_ref(), opts)?; + xvc_root_opt } XvcSubCommand::Pipeline(opts) => { let stdin = io::stdin(); let input = stdin.lock(); - Ok(pipeline::cmd_pipeline( + pipeline::cmd_pipeline( input, &output_snd, xvc_root_opt.as_ref().ok_or(Error::RequiresXvcRepository)?, opts, - )?) + )?; + + xvc_root_opt + } XvcSubCommand::CheckIgnore(opts) => { let stdin = io::stdin(); let input = stdin.lock(); - Ok(check_ignore::cmd_check_ignore( + check_ignore::cmd_check_ignore( input, &output_snd, xvc_root_opt.as_ref().ok_or(Error::RequiresXvcRepository)?, opts, - )?) + )?; + + xvc_root_opt + } XvcSubCommand::Storage(opts) => { let stdin = io::stdin(); let input = stdin.lock(); - Ok(storage::cmd_storage( + storage::cmd_storage( input, &output_snd, xvc_root_opt.as_ref().ok_or(Error::RequiresXvcRepository)?, opts, - )?) + )?; + + xvc_root_opt } - }?; + + }; watch!("Before handle_git_automation"); + match xvc_root_opt { - Some(xvc_root) => { - watch!(&cli_opts.command_string); + Some(ref xvc_root) => { + xvc_root.record(); if cli_opts.skip_git { debug!(output_snd, "Skipping Git operations"); } else { @@ -404,17 +397,102 @@ pub fn dispatch(cli_opts: cli::XvcCLI) -> Result<()> { ); } } - Ok(()) + Ok(xvc_root_opt) }); - match command_thread.join().unwrap() { + let xvc_root_opt = xvc_root_opt_res.join().unwrap(); + match &xvc_root_opt { Ok(_) => debug!(output_snd_clone, "Command completed successfully."), Err(e) => error!(output_snd_clone, "{}", e), } output_snd_clone.send(None).unwrap(); output_thread.join().unwrap(); + + xvc_root_opt }) .unwrap(); - Ok(()) + xvc_root_opt + + +} + +/// Dispatch commands to respective functions in the API +/// +/// It sets output verbosity with [XvcCLI::verbosity]. +/// Determines configuration sources by filling [XvcConfigInitParams]. +/// Tries to create an XvcRoot to determine whether we're inside one. +/// Creates two threads: One for running the API function, one for getting strings from output +/// channel. +/// +/// A corresponding function to reuse the same [XvcRoot] object is [test_dispatch]. +/// It doesn't recreate the whole configuration and this prevents errors regarding multiple +/// initializations. +pub fn dispatch(cli_opts: cli::XvcCLI) -> Result { + let verbosity = get_verbosity(&cli_opts); + + let term_log_level = get_term_log_level(verbosity); + + setup_logging( + Some(term_log_level), + if cli_opts.debug { + Some(LevelFilter::Trace) + } else { + None + }, + ); + + let xvc_config_params = get_xvc_config_params(&cli_opts); + + let xvc_root_opt = match load_xvc_root(xvc_config_params) { + Ok(r) => Some(r), + Err(e) => { + e.debug(); + None + } + }; + + dispatch_with_root(cli_opts, xvc_root_opt) + +} + +fn get_xvc_config_params(cli_opts: &XvcCLI) -> XvcConfigParams { + let xvc_config_params = XvcConfigParams { + current_dir: AbsolutePath::from(&cli_opts.workdir), + include_system_config: !cli_opts.no_system_config, + include_user_config: !cli_opts.no_user_config, + project_config_path: None, + local_config_path: None, + include_environment_config: !cli_opts.no_env_config, + command_line_config: Some(cli_opts.consolidate_config_options()), + default_configuration: default_project_config(true), + }; + xvc_config_params +} + +fn get_term_log_level(verbosity: XvcVerbosity) -> LevelFilter { + let term_log_level = match verbosity { + XvcVerbosity::Quiet => LevelFilter::Off, + XvcVerbosity::Default => LevelFilter::Error, + XvcVerbosity::Warn => LevelFilter::Warn, + XvcVerbosity::Info => LevelFilter::Info, + XvcVerbosity::Debug => LevelFilter::Debug, + XvcVerbosity::Trace => LevelFilter::Trace, + }; + term_log_level +} + +fn get_verbosity(cli_opts: &XvcCLI) -> XvcVerbosity { + let verbosity = if cli_opts.quiet { + XvcVerbosity::Quiet + } else { + match cli_opts.verbosity { + 0 => XvcVerbosity::Default, + 1 => XvcVerbosity::Warn, + 2 => XvcVerbosity::Info, + 3 => XvcVerbosity::Debug, + _ => XvcVerbosity::Trace, + } + }; + verbosity } diff --git a/lib/src/error.rs b/lib/src/error.rs index 4f730b7d5..32ba5192b 100644 --- a/lib/src/error.rs +++ b/lib/src/error.rs @@ -99,12 +99,12 @@ pub enum Error { #[from] source: io::Error, }, - #[error("Path is not in XVC Repository: {path:?}")] + #[error("Path is not in Xvc Repository: {path:?}")] PathNotInXvcRepository { path: OsString }, #[error("Path has no parent: {path:?}")] PathHasNoParent { path: OsString }, - #[error("This directory already belongs to an XVC repository {path:?}")] - DirectoryContainsXVCAlready { path: OsString }, + #[error("This directory already belongs to an Xvc repository {path:?}")] + DirectoryContainsXvcAlready { path: OsString }, #[error("This directory is not in a Git Repository {path:?}")] PathNotInGitRepository { path: OsString }, #[error("Cannot Parse Integer: {source:?}")] diff --git a/lib/src/git.rs b/lib/src/git.rs index 45cd18cf7..38cb29b39 100644 --- a/lib/src/git.rs +++ b/lib/src/git.rs @@ -111,7 +111,7 @@ pub fn git_checkout_ref( /// record the last entity counter before commit. pub fn handle_git_automation( output_snd: &XvcOutputSender, - xvc_root: XvcRoot, + xvc_root: &XvcRoot, to_branch: Option<&str>, xvc_cmd: &str, ) -> Result<()> { @@ -125,9 +125,6 @@ pub fn handle_git_automation( let xvc_dir = xvc_root.xvc_dir().clone(); let xvc_dir_str = xvc_dir.to_str().unwrap(); - // we drop here to record the final state - drop(xvc_root); - if use_git { if auto_commit { git_auto_commit( @@ -173,7 +170,7 @@ pub fn git_auto_commit( &[ "add", "--verbose", - &xvc_dir_str, + xvc_dir_str, "*.gitignore", "*.xvcignore", ], diff --git a/lib/src/init/mod.rs b/lib/src/init/mod.rs index d8d7965b4..e10c3309c 100644 --- a/lib/src/init/mod.rs +++ b/lib/src/init/mod.rs @@ -5,7 +5,7 @@ use log::{info, warn}; use std::env; use std::fs; use std::path::PathBuf; -use xvc_config::XvcConfigInitParams; +use xvc_config::XvcConfigParams; use xvc_core::default_project_config; use xvc_core::types::xvcroot::init_xvc_root; use xvc_core::util::git::inside_git; @@ -14,7 +14,7 @@ use xvc_logging::watch; use xvc_pipeline; use xvc_walker::AbsolutePath; -/// Initialize an XVC repository +/// Initialize an Xvc repository #[derive(Debug, Clone, Parser)] #[command(author, version)] pub struct InitCLI { @@ -59,7 +59,7 @@ pub fn run(xvc_root_opt: Option<&XvcRoot>, opts: InitCLI) -> Result { ); fs::remove_dir_all(xvc_root.xvc_dir())?; } else { - return Err(Error::DirectoryContainsXVCAlready { + return Err(Error::DirectoryContainsXvcAlready { path: xvc_root.absolute_path().as_os_str().to_os_string(), }); } @@ -84,7 +84,7 @@ pub fn run(xvc_root_opt: Option<&XvcRoot>, opts: InitCLI) -> Result { } } let default_configuration = default_project_config(!opts.no_git); - let config_opts = XvcConfigInitParams { + let config_opts = XvcConfigParams { default_configuration, current_dir: AbsolutePath::from(&path), include_system_config: true, diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 64bb7ff12..bbe8c8680 100755 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -1,6 +1,6 @@ #![warn(missing_docs)] #![forbid(unsafe_code)] -//! The main dispatching functions for the entire XVC CLI +//! The main dispatching functions for the entire Xvc CLI pub mod cli; pub mod error; pub mod git; @@ -11,7 +11,7 @@ pub mod api; pub use api::*; /// Adds `xvc` as the first elements to `args` and calls [cli::dispatch] after parsing them. -pub fn dispatch(args: Vec<&str>) -> Result<()> { +pub fn dispatch(args: Vec<&str>) -> Result { let args_with_binary_name = if !args.is_empty() && args[0] != "xvc" { vec!["xvc"].into_iter().chain(args).collect() } else { diff --git a/lib/src/main.rs b/lib/src/main.rs index 8f75088fa..7cab793ef 100755 --- a/lib/src/main.rs +++ b/lib/src/main.rs @@ -1,11 +1,13 @@ #![warn(missing_docs)] //! The entry point for xvc cli -use xvc::error::Result; +use xvc::{error::Result}; /// The entry point of the `xvc` cli. /// /// It parses the command line arguments [xvc::cli::XvcCLI] and calls [xvc::cli::dispatch] fn main() -> Result<()> { let cli_opts = xvc::cli::XvcCLI::from_args_os(std::env::args_os())?; - xvc::cli::dispatch(cli_opts) + xvc::cli::dispatch(cli_opts)?; + + Ok(()) } diff --git a/logging/Cargo.toml b/logging/Cargo.toml index b5a9e1608..3a7fe0a07 100644 --- a/logging/Cargo.toml +++ b/logging/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-logging" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Logging crate for Xvc" authors = ["Emre Şahin "] diff --git a/pipeline/Cargo.toml b/pipeline/Cargo.toml index e4174e0c8..15d111569 100644 --- a/pipeline/Cargo.toml +++ b/pipeline/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-pipeline" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Xvc data pipeline management" authors = ["Emre Şahin "] @@ -19,12 +19,12 @@ debug = true [dependencies] -xvc-config = { version = "0.6.7", path = "../config" } -xvc-core = { version = "0.6.7", path = "../core" } -xvc-ecs = { version = "0.6.7", path = "../ecs" } -xvc-logging = { version = "0.6.7", path = "../logging" } -xvc-walker = { version = "0.6.7", path = "../walker" } -xvc-file = { version = "0.6.7", path = "../file" } +xvc-config = { version = "0.6.8", path = "../config" } +xvc-core = { version = "0.6.8", path = "../core" } +xvc-ecs = { version = "0.6.8", path = "../ecs" } +xvc-logging = { version = "0.6.8", path = "../logging" } +xvc-walker = { version = "0.6.8", path = "../walker" } +xvc-file = { version = "0.6.8", path = "../file" } ## Cli and config clap = { version = "^4.4", features = ["derive"] } @@ -92,5 +92,5 @@ itertools = "^0.12" derive_more = "^0.99" [dev-dependencies] -xvc-test-helper = { version = "0.6.7", path = "../test_helper/" } +xvc-test-helper = { version = "0.6.8", path = "../test_helper/" } test-case = "^3.3" diff --git a/pipeline/src/lib.rs b/pipeline/src/lib.rs index 2b492c3fd..036af3c2d 100755 --- a/pipeline/src/lib.rs +++ b/pipeline/src/lib.rs @@ -52,7 +52,7 @@ use crate::pipeline::XvcStepInvalidate; pub use crate::pipeline::api::run::RunCLI; /// Pipeline management commands -#[derive(Debug, Parser)] +#[derive(Debug, Parser, Clone)] #[command(name = "pipeline")] pub struct PipelineCLI { /// Name of the pipeline this command applies to diff --git a/pipeline/src/pipeline/command.rs b/pipeline/src/pipeline/command.rs index 0bb138762..c2053f289 100644 --- a/pipeline/src/pipeline/command.rs +++ b/pipeline/src/pipeline/command.rs @@ -119,7 +119,7 @@ impl CommandProcess { stdout.read_to_string(&mut out)?; if !out.is_empty() { self.stdout_sender - .send(format!("[OUT] [{}] {} ", self.step.name, out)) + .send(format!("[OUT] [{}] {}", self.step.name, out)) .ok(); } } @@ -129,7 +129,7 @@ impl CommandProcess { stderr.read_to_string(&mut err)?; if !err.is_empty() { self.stderr_sender - .send(format!("[ERR] [{}] {} ", self.step.name, err)) + .send(format!("[ERR] [{}] {}", self.step.name, err)) .ok(); } } diff --git a/pipeline/src/pipeline/mod.rs b/pipeline/src/pipeline/mod.rs index cd57d22ba..77c9b5a09 100644 --- a/pipeline/src/pipeline/mod.rs +++ b/pipeline/src/pipeline/mod.rs @@ -240,7 +240,7 @@ struct StepThreadParams<'a> { output_diffs: Arc>>>, } -/// # XVC Pipeline Dependency Graph Rules +/// # Xvc Pipeline Dependency Graph Rules /// /// The dependency graph shows which steps of the pipeline depends on other /// steps. The dependency steps are set to run before the dependent steps. @@ -1257,19 +1257,19 @@ fn update_command_environment( if let Some(items) = actual.items() { match actual { XvcDependency::GlobItems(_) => { - update_env("XVC_GLOB_ADDED_ITEMS", &items)?; - update_env("XVC_GLOB_REMOVED_ITEMS", &[])?; - update_env("XVC_GLOB_ALL_ITEMS", &items) + update_env("XVC_ADDED_GLOB_ITEMS", &items)?; + update_env("XVC_REMOVED_GLOB_ITEMS", &[])?; + update_env("XVC_ALL_GLOB_ITEMS", &items) } XvcDependency::RegexItems(_) => { - update_env("XVC_REGEX_ADDED_ITEMS", &items)?; - update_env("XVC_REGEX_REMOVED_ITEMS", &[])?; - update_env("XVC_REGEX_ALL_ITEMS", &items) + update_env("XVC_ADDED_REGEX_ITEMS", &items)?; + update_env("XVC_REMOVED_REGEX_ITEMS", &[])?; + update_env("XVC_ALL_REGEX_ITEMS", &items) } XvcDependency::LineItems(_) => { - update_env("XVC_LINE_ADDED_ITEMS", &items)?; - update_env("XVC_LINE_REMOVED_ITEMS", &[])?; - update_env("XVC_LINE_ALL_ITEMS", &items) + update_env("XVC_ADDED_ITEMS", &items)?; + update_env("XVC_REMOVED_LINE_ITEMS", &[])?; + update_env("XVC_ALL_LINE_ITEMS", &items) } _ => Ok(()), } @@ -1282,19 +1282,19 @@ fn update_command_environment( if let Some(items) = record.items() { match record { XvcDependency::GlobItems(_dep) => { - update_env("XVC_GLOB_ADDED_ITEMS", &[])?; - update_env("XVC_GLOB_REMOVED_ITEMS", &items)?; - update_env("XVC_GLOB_ALL_ITEMS", &items) + update_env("XVC_ADDED_GLOB_ITEMS", &[])?; + update_env("XVC_REMOVED_GLOB_ITEMS", &items)?; + update_env("XVC_ALL_GLOB_ITEMS", &items) } XvcDependency::RegexItems(_dep) => { - update_env("XVC_REGEX_ADDED_ITEMS", &[])?; - update_env("XVC_REGEX_REMOVED_ITEMS", &items)?; - update_env("XVC_REGEX_ALL_ITEMS", &items) + update_env("XVC_ADDED_REGEX_ITEMS", &[])?; + update_env("XVC_REMOVED_REGEX_ITEMS", &items)?; + update_env("XVC_ALL_REGEX_ITEMS", &items) } XvcDependency::LineItems(_dep) => { - update_env("XVC_LINE_ADDED_ITEMS", &[])?; - update_env("XVC_LINE_REMOVED_ITEMS", &items)?; - update_env("XVC_LINE_ALL_ITEMS", &items) + update_env("XVC_ADDED_LINE_ITEMS", &[])?; + update_env("XVC_REMOVED_LINE_ITEMS", &items)?; + update_env("XVC_ALL_LINE_ITEMS", &items) } _ => Ok(()), } @@ -1340,22 +1340,22 @@ fn update_command_environment( } } } - update_env("XVC_GLOB_CHANGED_ITEMS", &changed_items)?; + update_env("XVC_CHANGED_GLOB_ITEMS", &changed_items)?; } - update_env("XVC_GLOB_ADDED_ITEMS", &added_items)?; - update_env("XVC_GLOB_REMOVED_ITEMS", &removed_items)?; - update_env("XVC_GLOB_ALL_ITEMS", &all_items) + update_env("XVC_ADDED_GLOB_ITEMS", &added_items)?; + update_env("XVC_REMOVED_GLOB_ITEMS", &removed_items)?; + update_env("XVC_ALL_GLOB_ITEMS", &all_items) } XvcDependency::RegexItems(_dep) => { - update_env("XVC_REGEX_ADDED_ITEMS", &added_items)?; - update_env("XVC_REGEX_REMOVED_ITEMS", &removed_items)?; - update_env("XVC_REGEX_ALL_ITEMS", &all_items) + update_env("XVC_ADDED_REGEX_ITEMS", &added_items)?; + update_env("XVC_REMOVED_REGEX_ITEMS", &removed_items)?; + update_env("XVC_ALL_REGEX_ITEMS", &all_items) } XvcDependency::LineItems(_dep) => { - update_env("XVC_LINE_ADDED_ITEMS", &added_items)?; - update_env("XVC_LINE_REMOVED_ITEMS", &removed_items)?; - update_env("XVC_LINE_ALL_ITEMS", &all_items) + update_env("XVC_ADDED_LINE_ITEMS", &added_items)?; + update_env("XVC_REMOVED_LINE_ITEMS", &removed_items)?; + update_env("XVC_ALL_LINE_ITEMS", &all_items) } _ => Ok(()), } @@ -1371,19 +1371,19 @@ fn update_command_environment( if let Some(items) = dep.items() { match dep { XvcDependency::GlobItems(_) => { - update_env("XVC_GLOB_ADDED_ITEMS", &[])?; - update_env("XVC_GLOB_REMOVED_ITEMS", &[])?; - update_env("XVC_GLOB_ALL_ITEMS", &items) + update_env("XVC_ADDED_GLOB_ITEMS", &[])?; + update_env("XVC_REMOVED_GLOB_ITEMS", &[])?; + update_env("XVC_ALL_GLOB_ITEMS", &items) } XvcDependency::RegexItems(_) => { - update_env("XVC_REGEX_ADDED_ITEMS", &[])?; - update_env("XVC_REGEX_REMOVED_ITEMS", &[])?; - update_env("XVC_REGEX_ALL_ITEMS", &items) + update_env("XVC_ADDED_REGEX_ITEMS", &[])?; + update_env("XVC_REMOVED_REGEX_ITEMS", &[])?; + update_env("XVC_ALL_REGEX_ITEMS", &items) } XvcDependency::LineItems(_) => { - update_env("XVC_LINE_ADDED_ITEMS", &[])?; - update_env("XVC_LINE_REMOVED_ITEMS", &[])?; - update_env("XVC_LINE_ALL_ITEMS", &items) + update_env("XVC_ADDED_LINE_ITEMS", &[])?; + update_env("XVC_REMOVED_LINE_ITEMS", &[])?; + update_env("XVC_ALL_LINE_ITEMS", &items) } _ => Ok(()), } @@ -1488,7 +1488,7 @@ fn s_running_f_wait_process<'a>( Some(exit_code) => match exit_code { ExitStatus::Exited(0) => { - output!(params.output_snd, "[DONE] {} ({})", step.name, step_command); + output!(params.output_snd, "[DONE] {} ({})\n", step.name, step_command); return_state = Some(s.process_completed_successfully()); } , diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 6ab74b461..b602988c8 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "xvc-storage" -version = "0.6.7" +version = "0.6.8" edition = "2021" -description = "Xvc remote (and local) storage management" +description = "Xvc remote and local storage management" authors = ["Emre Şahin "] license = "GPL-3.0" homepage = "https://xvc.dev" @@ -20,11 +20,11 @@ debug = true [dependencies] -xvc-logging = { version = "0.6.7", path = "../logging" } -xvc-config = { version = "0.6.7", path = "../config" } -xvc-core = { version = "0.6.7", path = "../core" } -xvc-ecs = { version = "0.6.7", path = "../ecs" } -xvc-walker = { version = "0.6.7", path = "../walker" } +xvc-logging = { version = "0.6.8", path = "../logging" } +xvc-config = { version = "0.6.8", path = "../config" } +xvc-core = { version = "0.6.8", path = "../core" } +xvc-ecs = { version = "0.6.8", path = "../ecs" } +xvc-walker = { version = "0.6.8", path = "../walker" } ## Cli and config clap = { version = "^4.4", features = ["derive"] } @@ -109,7 +109,7 @@ digital-ocean = ["async"] [dev-dependencies] -xvc-test-helper = { version = "0.6.7", path = "../test_helper/" } +xvc-test-helper = { version = "0.6.8", path = "../test_helper/" } shellfn = "^0.1" [package.metadata.cargo-udeps.ignore] diff --git a/storage/src/error.rs b/storage/src/error.rs index 0dcbf9264..f653ab1d4 100644 --- a/storage/src/error.rs +++ b/storage/src/error.rs @@ -61,7 +61,7 @@ pub enum Error { NoRepositoryGuidFound, #[error("Cannot find remote with identifier: {identifier}")] - CannotFindRemoteWithIdentifier { identifier: StorageIdentifier }, + CannotFindStorageWithIdentifier { identifier: StorageIdentifier }, #[error("Process Exec Error: {source}")] ProcessExecError { diff --git a/storage/src/lib.rs b/storage/src/lib.rs index b8c339ca6..2d2c4ab99 100755 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -25,7 +25,7 @@ use xvc_core::XvcRoot; use xvc_logging::{output, XvcOutputSender}; /// Storage (on the cloud) management commands -#[derive(Debug, Parser)] +#[derive(Debug, Parser, Clone)] #[command(name = "storage", about = "")] pub struct StorageCLI { /// Subcommand for storage management @@ -78,7 +78,7 @@ pub enum StorageNewSubCommand { /// Add a new generic storage. /// /// ⚠️ Please note that this is an advanced method to configure storages. - /// You may damage your repository and local and remote files with incorrect configurations. + /// You may damage your repository and local and storage files with incorrect configurations. /// /// Please see https://docs.xvc.dev/ref/xvc-storage-new-generic.html for examples and make /// necessary backups. @@ -125,12 +125,10 @@ pub enum StorageNewSubCommand { storage_dir: Option, }, - /// Add a new rsync storage - /// - /// Uses rsync in separate processes to communicate with the server. + /// Add a new rsync storages /// + /// Uses rsync in separate processes to communicate. /// This can be used when you already have an SSH/Rsync connection. - /// /// It doesn't prompt for any passwords. The connection must be set up with ssh keys beforehand. #[command()] Rsync { diff --git a/storage/src/storage/async_common.rs b/storage/src/storage/async_common.rs index 3f7f6e66e..43df2ab77 100644 --- a/storage/src/storage/async_common.rs +++ b/storage/src/storage/async_common.rs @@ -31,28 +31,28 @@ use super::XvcStorageTempDir; use super::XVC_STORAGE_GUID_FILENAME; pub trait XvcS3StorageOperations { - fn remote_prefix(&self) -> String; + fn storage_prefix(&self) -> String; fn guid(&self) -> &XvcStorageGuid; fn get_bucket(&self) -> Result; fn credentials(&self) -> Result; fn bucket_name(&self) -> String; - fn build_remote_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { + fn build_storage_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { XvcStoragePath::from(format!( "{}/{}/{}", - self.remote_prefix(), + self.storage_prefix(), self.guid(), cache_path )) } fn region(&self) -> String; - async fn write_remote_guid(&self) -> Result<()> { + async fn write_storage_guid(&self) -> Result<()> { let guid_str = self.guid().to_string(); let guid_bytes = guid_str.as_bytes(); let bucket = self.get_bucket()?; let response = bucket .put_object( - format!("{}/{}", self.remote_prefix(), XVC_STORAGE_GUID_FILENAME), + format!("{}/{}", self.storage_prefix(), XVC_STORAGE_GUID_FILENAME), guid_bytes, ) .await; @@ -64,7 +64,7 @@ pub trait XvcS3StorageOperations { } async fn a_init(&mut self, output_snd: &XvcOutputSender) -> Result { - let res_response = self.write_remote_guid().await; + let res_response = self.write_storage_guid().await; let guid = self.guid().clone(); @@ -86,11 +86,11 @@ pub trait XvcS3StorageOperations { let region = Region::from_str(&self.region()).unwrap_or("us-east-1".parse().unwrap()); let bucket = Bucket::new(&self.bucket_name(), region, credentials)?; let xvc_guid = xvc_root.config().guid().unwrap(); - let prefix = self.remote_prefix().clone(); + let prefix = self.storage_prefix().clone(); let res_list = bucket .list( - format!("{}/{}", self.remote_prefix(), xvc_guid), + format!("{}/{}", self.storage_prefix(), xvc_guid), Some("/".to_string()), ) .await; @@ -143,7 +143,7 @@ pub trait XvcS3StorageOperations { for cache_path in paths { watch!(cache_path); - let remote_path = self.build_remote_path(cache_path); + let storage_path = self.build_storage_path(cache_path); let abs_cache_path = cache_path.to_absolute_path(xvc_root); watch!(abs_cache_path); @@ -151,13 +151,13 @@ pub trait XvcS3StorageOperations { watch!(path); let res_response = bucket - .put_object_stream(&mut path, remote_path.as_str()) + .put_object_stream(&mut path, storage_path.as_str()) .await; match res_response { Ok(_) => { - info!(output_snd, "{} -> {}", abs_cache_path, remote_path.as_str()); - copied_paths.push(remote_path); + info!(output_snd, "{} -> {}", abs_cache_path, storage_path.as_str()); + copied_paths.push(storage_path); watch!(copied_paths.len()); } Err(err) => { @@ -185,21 +185,21 @@ pub trait XvcS3StorageOperations { for cache_path in paths { watch!(cache_path); - let remote_path = self.build_remote_path(cache_path); + let storage_path = self.build_storage_path(cache_path); let abs_cache_dir = temp_dir.temp_cache_dir(cache_path)?; fs::create_dir_all(&abs_cache_dir)?; let abs_cache_path = temp_dir.temp_cache_path(cache_path)?; watch!(abs_cache_path); - let response_data_stream = bucket.get_object_stream(remote_path.as_str()).await; + let response_data_stream = bucket.get_object_stream(storage_path.as_str()).await; match response_data_stream { Ok(mut response) => { - info!(output_snd, "{} -> {}", remote_path.as_str(), abs_cache_path); + info!(output_snd, "{} -> {}", storage_path.as_str(), abs_cache_path); let mut async_cache_path = tokio::fs::File::create(&abs_cache_path).await?; while let Some(chunk) = response.bytes().next().await { async_cache_path.write_all(&chunk).await?; } - copied_paths.push(remote_path); + copied_paths.push(storage_path); watch!(copied_paths.len()); } Err(err) => { @@ -228,10 +228,10 @@ pub trait XvcS3StorageOperations { for cache_path in paths { watch!(cache_path); - let remote_path = self.build_remote_path(cache_path); - bucket.delete_object(remote_path.as_str()).await?; - info!(output, "[DELETE] {}", remote_path.as_str()); - deleted_paths.push(remote_path); + let storage_path = self.build_storage_path(cache_path); + bucket.delete_object(storage_path.as_str()).await?; + info!(output, "[DELETE] {}", storage_path.as_str()); + deleted_paths.push(storage_path); } Ok(XvcStorageDeleteEvent { @@ -256,7 +256,7 @@ pub trait XvcS3StorageOperations { // let expiration_seconds = duration.as_secs() as u32; - let path = self.build_remote_path(path); + let path = self.build_storage_path(path); let signed_url = bucket.presign_get(path.as_str(), expiration_seconds, None)?; info!(output, "[SHARED] {}", path.as_str()); output!(output, "{}", signed_url); diff --git a/storage/src/storage/digital_ocean.rs b/storage/src/storage/digital_ocean.rs index 006b198fc..32cea6ec5 100644 --- a/storage/src/storage/digital_ocean.rs +++ b/storage/src/storage/digital_ocean.rs @@ -14,9 +14,9 @@ use crate::{XvcStorageGuid, XvcStorageOperations}; use super::async_common::XvcS3StorageOperations; -/// Configure a new Digital Ocean Spaces remote. +/// Configure a new Digital Ocean Spaces storage. /// -/// `bucket_name`, `region` and `remote_prefix` sets a URL for the storage +/// `bucket_name`, `region` and `storage_prefix` sets a URL for the storage /// location. /// /// This creates a [XvcDigitalOceanStorage], calls its @@ -29,18 +29,18 @@ pub fn cmd_new_digital_ocean( name: String, bucket_name: String, region: String, - remote_prefix: String, + storage_prefix: String, ) -> Result<()> { - let mut remote = XvcDigitalOceanStorage { + let mut storage = XvcDigitalOceanStorage { guid: XvcStorageGuid::new(), name, region, bucket_name, - remote_prefix, + storage_prefix, }; - watch!(remote); + watch!(storage); - let init_event = remote.init(output_snd, xvc_root)?; + let init_event = storage.init(output_snd, xvc_root)?; watch!(init_event); xvc_root.with_r1nstore_mut(|store: &mut R1NStore| { @@ -48,7 +48,7 @@ pub fn cmd_new_digital_ocean( let event_e = xvc_root.new_entity(); store.insert( store_e, - XvcStorage::DigitalOcean(remote.clone()), + XvcStorage::DigitalOcean(storage.clone()), event_e, XvcStorageEvent::Init(init_event.clone()), ); @@ -58,23 +58,23 @@ pub fn cmd_new_digital_ocean( Ok(()) } -/// A Digital Ocean Spaces remote. +/// A Digital Ocean Spaces storage. #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] pub struct XvcDigitalOceanStorage { - /// The GUID of the remote. + /// The GUID of the storage. pub guid: XvcStorageGuid, - /// The name of the remote. + /// The name of the storage. pub name: String, - /// The region of the remote. + /// The region of the storage. pub region: String, - /// The bucket name of the remote. + /// The bucket name of the storage. pub bucket_name: String, - /// The remote prefix of the remote. - pub remote_prefix: String, + /// The path prefix of the storage. + pub storage_prefix: String, } impl XvcDigitalOceanStorage { - fn remote_specific_credentials(&self) -> Result { + fn storage_specific_credentials(&self) -> Result { Credentials::new( Some(&env::var(format!( "XVC_STORAGE_ACCESS_KEY_ID_{}", @@ -101,8 +101,8 @@ impl XvcDigitalOceanStorage { } impl XvcS3StorageOperations for XvcDigitalOceanStorage { - fn remote_prefix(&self) -> String { - self.remote_prefix.clone() + fn storage_prefix(&self) -> String { + self.storage_prefix.clone() } fn guid(&self) -> &XvcStorageGuid { @@ -110,7 +110,7 @@ impl XvcS3StorageOperations for XvcDigitalOceanStorage { } fn credentials(&self) -> Result { - match self.remote_specific_credentials() { + match self.storage_specific_credentials() { Ok(c) => Ok(c), Err(e1) => match self.storage_type_credentials() { Ok(c) => Ok(c), diff --git a/storage/src/storage/gcs.rs b/storage/src/storage/gcs.rs index 48225da19..3ea5a429d 100644 --- a/storage/src/storage/gcs.rs +++ b/storage/src/storage/gcs.rs @@ -16,7 +16,7 @@ use super::async_common::XvcS3StorageOperations; /// Configure a new Google Cloud Storage remote. /// -/// `bucket_name`, `region` and `remote_prefix` sets a URL for the storage +/// `bucket_name`, `region` and `storage_prefix` sets a URL for the storage /// location. /// /// This creates a [XvcGcsStorage], calls its @@ -29,19 +29,19 @@ pub fn cmd_new_gcs( name: String, bucket_name: String, region: String, - remote_prefix: String, + storage_prefix: String, ) -> Result<()> { - let mut remote = XvcGcsStorage { + let mut storage = XvcGcsStorage { guid: XvcStorageGuid::new(), name, region, bucket_name, - remote_prefix, + storage_prefix, }; - watch!(remote); + watch!(storage); - let init_event = remote.init(output_snd, xvc_root)?; + let init_event = storage.init(output_snd, xvc_root)?; watch!(init_event); xvc_root.with_r1nstore_mut(|store: &mut R1NStore| { @@ -49,7 +49,7 @@ pub fn cmd_new_gcs( let event_e = xvc_root.new_entity(); store.insert( store_e, - XvcStorage::Gcs(remote.clone()), + XvcStorage::Gcs(storage.clone()), event_e, XvcStorageEvent::Init(init_event.clone()), ); @@ -62,20 +62,20 @@ pub fn cmd_new_gcs( /// A Google Cloud Storage remote. #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] pub struct XvcGcsStorage { - /// The unique identifier of the remote. + /// The unique identifier of the storage. pub guid: XvcStorageGuid, - /// The name of the remote. + /// The name of the storage. pub name: String, - /// The region of the remote. + /// The region of the storage. pub region: String, - /// The bucket name of the remote. + /// The bucket name of the storage. pub bucket_name: String, - /// The remote prefix of the remote. - pub remote_prefix: String, + /// The path prefix on the storage. + pub storage_prefix: String, } impl XvcGcsStorage { - fn remote_specific_credentials(&self) -> Result { + fn storage_specific_credentials(&self) -> Result { Credentials::new( Some(&env::var(format!( "XVC_STORAGE_ACCESS_KEY_ID_{}", @@ -102,8 +102,8 @@ impl XvcGcsStorage { } impl XvcS3StorageOperations for XvcGcsStorage { - fn remote_prefix(&self) -> String { - self.remote_prefix.clone() + fn storage_prefix(&self) -> String { + self.storage_prefix.clone() } fn guid(&self) -> &XvcStorageGuid { @@ -129,7 +129,7 @@ impl XvcS3StorageOperations for XvcGcsStorage { } fn credentials(&self) -> Result { - match self.remote_specific_credentials() { + match self.storage_specific_credentials() { Ok(c) => Ok(c), Err(e1) => match self.storage_type_credentials() { Ok(c) => Ok(c), @@ -143,289 +143,3 @@ impl XvcS3StorageOperations for XvcGcsStorage { .map_err(|e| e.into()) } } -// async fn a_init( -// self, -// output_snd: &XvcOutputSender, -// _xvc_root: &xvc_core::XvcRoot, -// ) -> Result<(XvcStorageInitEvent, Self)> { -// let bucket = self.get_bucket()?; -// let guid = self.guid.clone(); -// let guid_str = self.guid.to_string(); -// let guid_bytes = guid_str.as_bytes(); -// -// watch!(bucket); -// watch!(guid); -// watch!(guid_str); -// -// let res_response = bucket -// .put_object( -// format!("{}/{}", self.remote_prefix, XVC_STORAGE_GUID_FILENAME), -// guid_bytes, -// ) -// .await; -// -// match res_response { -// Ok(_) => Ok((XvcStorageInitEvent { guid }, self)), -// Err(err) => { -// error!(output_snd, "Error: {}", err); -// Err(Error::S3Error { source: err }) -// } -// } -// } -// -// async fn a_list( -// &self, -// output_snd: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// ) -> Result { -// let credentials = self.credentials()?; -// let region = Region::from_str(&self.region).unwrap_or("us-east-1".parse().unwrap()); -// let bucket = Bucket::new(&self.bucket_name, region, credentials)?; -// let _guid = self.guid.clone(); -// let guid_str = self.guid.to_string(); -// let _guid_bytes = guid_str.as_bytes(); -// let xvc_guid = xvc_root.config().guid().unwrap(); -// let prefix = self.remote_prefix.clone(); -// -// let res_list = bucket -// .list( -// format!("{}/{}", self.remote_prefix, xvc_guid), -// Some("/".to_string()), -// ) -// .await; -// -// match res_list { -// Ok(list_all) => { -// // select only the matching elements -// let re = Regex::new(&format!( -// "{prefix}/{xvc_guid}/{cp}/{d3}/{d3}/{d58}/0\\..*$", -// cp = r#"[a-zA-Z][0-9]"#, -// d3 = r#"[0-9A-Fa-f]{3}"#, -// d58 = r#"[0-9A-Fa-f]{58}"# -// )) -// .unwrap(); -// -// let paths = list_all -// .iter() -// .filter_map(|e| { -// if re.is_match(e.name.as_ref()) { -// Some(XvcStoragePath::from_str(&e.name).unwrap()) -// } else { -// None -// } -// }) -// .collect(); -// -// Ok(XvcStorageListEvent { -// guid: self.guid.clone(), -// paths, -// }) -// } -// -// Err(err) => { -// error!(output_snd, "Error: {}", err); -// Err(Error::S3Error { source: err }) -// } -// } -// } -// -// fn build_remote_path(&self, repo_guid: &str, cache_path: &XvcCachePath) -> XvcStoragePath { -// XvcStoragePath::from(format!( -// "{}/{}/{}", -// self.remote_prefix, repo_guid, cache_path -// )) -// } -// -// async fn a_send( -// &self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// paths: &[xvc_core::XvcCachePath], -// _force: bool, -// ) -> crate::Result { -// let repo_guid = xvc_root -// .config() -// .guid() -// .ok_or_else(|| crate::Error::NoRepositoryGuidFound)?; -// let mut copied_paths = Vec::::new(); -// -// let bucket = self.get_bucket()?; -// -// for cache_path in paths { -// watch!(cache_path); -// let remote_path = self.build_remote_path(&repo_guid, cache_path); -// let abs_cache_path = cache_path.to_absolute_path(xvc_root); -// watch!(abs_cache_path); -// -// let mut path = tokio::fs::File::open(&abs_cache_path).await?; -// watch!(path); -// -// let res_response = bucket -// .put_object_stream(&mut path, remote_path.as_str()) -// .await; -// -// match res_response { -// Ok(_) => { -// info!(output, "{} -> {}", abs_cache_path, remote_path.as_str()); -// copied_paths.push(remote_path); -// watch!(copied_paths.len()); -// } -// Err(err) => { -// error!(output, "Error: {}", err); -// } -// } -// } -// -// Ok(XvcStorageSendEvent { -// guid: self.guid.clone(), -// paths: copied_paths, -// }) -// } -// -// async fn a_receive( -// &self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// paths: &[xvc_core::XvcCachePath], -// _force: bool, -// ) -> Result<(XvcStorageTempDir, XvcStorageReceiveEvent)> { -// let repo_guid = xvc_root -// .config() -// .guid() -// .ok_or_else(|| crate::Error::NoRepositoryGuidFound)?; -// let mut copied_paths = Vec::::new(); -// -// let bucket = self.get_bucket()?; -// let temp_dir = XvcStorageTempDir::new()?; -// -// for cache_path in paths { -// watch!(cache_path); -// let remote_path = self.build_remote_path(&repo_guid, cache_path); -// let abs_cache_dir = temp_dir.temp_cache_dir(cache_path)?; -// fs::create_dir_all(&abs_cache_dir)?; -// let abs_cache_path = temp_dir.temp_cache_path(cache_path)?; -// watch!(abs_cache_path); -// -// let response_data_stream = bucket.get_object_stream(remote_path.as_str()).await; -// -// match response_data_stream { -// Ok(mut response) => { -// info!(output, "{} -> {}", remote_path.as_str(), abs_cache_path); -// let mut async_cache_path = tokio::fs::File::create(&abs_cache_path).await?; -// while let Some(chunk) = response.bytes().next().await { -// async_cache_path.write_all(&chunk).await?; -// } -// copied_paths.push(remote_path); -// watch!(copied_paths.len()); -// } -// Err(err) => { -// error!(output, "{}", err); -// } -// } -// } -// -// Ok(( -// temp_dir, -// XvcStorageReceiveEvent { -// guid: self.guid.clone(), -// paths: copied_paths, -// }, -// )) -// } -// -// async fn a_delete( -// &self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// paths: &[XvcCachePath], -// ) -> Result { -// let repo_guid = xvc_root -// .config() -// .guid() -// .ok_or_else(|| crate::Error::NoRepositoryGuidFound)?; -// let mut deleted_paths = Vec::::new(); -// -// let bucket = self.get_bucket()?; -// -// for cache_path in paths { -// watch!(cache_path); -// let remote_path = self.build_remote_path(&repo_guid, cache_path); -// bucket.delete_object(remote_path.as_str()).await?; -// info!(output, "[DELETE] {}", remote_path.as_str()); -// deleted_paths.push(remote_path); -// } -// -// Ok(XvcStorageDeleteEvent { -// guid: self.guid.clone(), -// paths: deleted_paths, -// }) -// } -// } -// -// impl XvcStorageOperations for XvcGcsStorage { -// fn init( -// self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// ) -> Result<(XvcStorageInitEvent, Self)> { -// let rt = tokio::runtime::Builder::new_multi_thread() -// .enable_all() -// .build()?; -// watch!(rt); -// rt.block_on(self.a_init(output, xvc_root)) -// } -// -// /// List the bucket contents that start with `self.remote_prefix` -// fn list( -// &self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// ) -> crate::Result { -// let rt = tokio::runtime::Builder::new_multi_thread() -// .enable_all() -// .build() -// .unwrap(); -// rt.block_on(self.a_list(output, xvc_root)) -// } -// -// fn send( -// &self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// paths: &[xvc_core::XvcCachePath], -// force: bool, -// ) -> crate::Result { -// let rt = tokio::runtime::Builder::new_multi_thread() -// .enable_all() -// .build() -// .unwrap(); -// rt.block_on(self.a_send(output, xvc_root, paths, force)) -// } -// -// fn receive( -// &self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// paths: &[xvc_core::XvcCachePath], -// force: bool, -// ) -> crate::Result<(XvcStorageTempDir, XvcStorageReceiveEvent)> { -// let rt = tokio::runtime::Builder::new_multi_thread() -// .enable_all() -// .build() -// .unwrap(); -// rt.block_on(self.a_receive(output, xvc_root, paths, force)) -// } -// -// fn delete( -// &self, -// output: &XvcOutputSender, -// xvc_root: &xvc_core::XvcRoot, -// paths: &[xvc_core::XvcCachePath], -// ) -> crate::Result { -// let rt = tokio::runtime::Builder::new_multi_thread() -// .enable_all() -// .build() -// .unwrap(); -// rt.block_on(self.a_delete(output, xvc_root, paths)) -// } -// } diff --git a/storage/src/storage/local.rs b/storage/src/storage/local.rs index 1a8e7de63..03346198a 100644 --- a/storage/src/storage/local.rs +++ b/storage/src/storage/local.rs @@ -26,19 +26,19 @@ pub fn cmd_storage_new_local( path: PathBuf, name: String, ) -> Result<()> { - let mut remote = XvcLocalStorage { + let mut storage = XvcLocalStorage { guid: XvcStorageGuid::new(), name, path, }; - let init_event = remote.init(output_snd, xvc_root)?; + let init_event = storage.init(output_snd, xvc_root)?; xvc_root.with_r1nstore_mut(|store: &mut R1NStore| { let store_e = xvc_root.new_entity(); let event_e = xvc_root.new_entity(); store.insert( store_e, - XvcStorage::Local(remote.clone()), + XvcStorage::Local(storage.clone()), event_e, XvcStorageEvent::Init(init_event.clone()), ); @@ -60,7 +60,7 @@ pub struct XvcLocalStorage { } impl XvcLocalStorage { - fn remote_path(&self, repo_guid: &str, cache_path: &XvcCachePath) -> XvcStoragePath { + fn storage_path(&self, repo_guid: &str, cache_path: &XvcCachePath) -> XvcStoragePath { XvcStoragePath::from(format!("{}/{}", repo_guid, cache_path)) } } @@ -99,7 +99,7 @@ impl XvcStorageOperations for XvcLocalStorage { fs::write(guid_filename, format!("{}", self.guid))?; info!( output, - "Created local remote directory {} with guid: {}", + "Created local storage directory {} with guid: {}", self.path.to_string_lossy(), self.guid ); @@ -127,24 +127,24 @@ impl XvcStorageOperations for XvcLocalStorage { let mut copied_paths = Vec::::new(); for cache_path in paths { - let remote_path = self.remote_path(&repo_guid, cache_path); - let abs_remote_path = remote_path.as_ref().to_logical_path(&self.path); - if force && abs_remote_path.exists() { - fs::remove_file(abs_remote_path.clone())?; + let storage_path = self.storage_path(&repo_guid, cache_path); + let abs_storage_path = storage_path.as_ref().to_logical_path(&self.path); + if force && abs_storage_path.exists() { + fs::remove_file(abs_storage_path.clone())?; } else { - info!(output, "[SKIPPED] {}", remote_path) + info!(output, "[SKIPPED] {}", storage_path) } let abs_cache_path = cache_path.to_absolute_path(xvc_root); - let abs_remote_dir = abs_remote_path.parent().unwrap(); - fs::create_dir_all(abs_remote_dir)?; - fs::copy(&abs_cache_path, &abs_remote_path)?; - copied_paths.push(remote_path); + let abs_storage_dir = abs_storage_path.parent().unwrap(); + fs::create_dir_all(abs_storage_dir)?; + fs::copy(&abs_cache_path, &abs_storage_path)?; + copied_paths.push(storage_path); watch!(copied_paths.len()); info!( output, "{} -> {}", abs_cache_path, - abs_remote_path.to_string_lossy() + abs_storage_path.to_string_lossy() ); } @@ -169,19 +169,19 @@ impl XvcStorageOperations for XvcLocalStorage { let temp_dir = XvcStorageTempDir::new()?; for cache_path in paths { - let remote_path = self.remote_path(&repo_guid, cache_path); - let abs_remote_path = remote_path.as_ref().to_logical_path(&self.path); + let storage_path = self.storage_path(&repo_guid, cache_path); + let abs_storage_path = storage_path.as_ref().to_logical_path(&self.path); let abs_cache_path = temp_dir.temp_cache_path(cache_path)?; let abs_cache_dir = temp_dir.temp_cache_dir(cache_path)?; fs::create_dir_all(&abs_cache_dir)?; - fs::copy(&abs_remote_path, &abs_cache_path)?; + fs::copy(&abs_storage_path, &abs_cache_path)?; watch!(abs_cache_path.exists()); - copied_paths.push(remote_path); + copied_paths.push(storage_path); watch!(copied_paths.len()); info!( output, "{} -> {}", - abs_remote_path.to_string_lossy(), + abs_storage_path.to_string_lossy(), abs_cache_path ); } @@ -208,11 +208,11 @@ impl XvcStorageOperations for XvcLocalStorage { let mut deleted_paths = Vec::::new(); for cache_path in paths { - let remote_path = self.remote_path(&repo_guid, cache_path); - let abs_remote_path = remote_path.as_ref().to_logical_path(&self.path); - fs::remove_file(&abs_remote_path)?; - info!(output, "[DELETE] {}", abs_remote_path.to_string_lossy()); - deleted_paths.push(remote_path); + let storage_path = self.storage_path(&repo_guid, cache_path); + let abs_storage_path = storage_path.as_ref().to_logical_path(&self.path); + fs::remove_file(&abs_storage_path)?; + info!(output, "[DELETE] {}", abs_storage_path.to_string_lossy()); + deleted_paths.push(storage_path); } Ok(XvcStorageDeleteEvent { diff --git a/storage/src/storage/minio.rs b/storage/src/storage/minio.rs index 748d7d591..bc26f3ba7 100644 --- a/storage/src/storage/minio.rs +++ b/storage/src/storage/minio.rs @@ -18,7 +18,7 @@ use super::XvcStoragePath; /// Configure a new Minio remote storage. /// -/// `endpoint`, `bucket_name`, `region` and `remote_prefix` sets a URL for the +/// `endpoint`, `bucket_name`, `region` and `storage_prefix` sets a URL for the /// storage location. /// /// This creates a [XvcMinioStorage], calls its @@ -34,19 +34,19 @@ pub fn cmd_new_minio( endpoint: String, bucket_name: String, region: String, - remote_prefix: String, + storage_prefix: String, ) -> Result<()> { - let mut remote = XvcMinioStorage { + let mut storage = XvcMinioStorage { guid: XvcStorageGuid::new(), name, region, bucket_name, - remote_prefix, + storage_prefix, endpoint, }; - watch!(remote); + watch!(storage); - let init_event = remote.init(output_snd, xvc_root)?; + let init_event = storage.init(output_snd, xvc_root)?; watch!(init_event); xvc_root.with_r1nstore_mut(|store: &mut R1NStore| { @@ -54,7 +54,7 @@ pub fn cmd_new_minio( let event_e = xvc_root.new_entity(); store.insert( store_e, - XvcStorage::Minio(remote.clone()), + XvcStorage::Minio(storage.clone()), event_e, XvcStorageEvent::Init(init_event.clone()), ); @@ -76,13 +76,13 @@ pub struct XvcMinioStorage { /// Bucket name of the storage pub bucket_name: String, /// Prefix of the storage within the bucket_name - pub remote_prefix: String, + pub storage_prefix: String, /// Full endpoint of the storage pub endpoint: String, } impl XvcMinioStorage { - fn remote_specific_credentials(&self) -> Result { + fn storage_specific_credentials(&self) -> Result { Credentials::new( Some(&env::var(format!( "XVC_STORAGE_ACCESS_KEY_ID_{}", @@ -109,8 +109,8 @@ impl XvcMinioStorage { } impl XvcS3StorageOperations for XvcMinioStorage { - fn remote_prefix(&self) -> String { - self.remote_prefix.clone() + fn storage_prefix(&self) -> String { + self.storage_prefix.clone() } fn guid(&self) -> &XvcStorageGuid { @@ -129,7 +129,7 @@ impl XvcS3StorageOperations for XvcMinioStorage { } fn credentials(&self) -> Result { - match self.remote_specific_credentials() { + match self.storage_specific_credentials() { Ok(c) => Ok(c), Err(e1) => match self.storage_type_credentials() { Ok(c) => Ok(c), @@ -147,10 +147,10 @@ impl XvcS3StorageOperations for XvcMinioStorage { self.bucket_name.clone() } - fn build_remote_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { + fn build_storage_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { XvcStoragePath::from(format!( "{}/{}/{}", - self.remote_prefix, + self.storage_prefix, self.guid(), cache_path )) @@ -159,223 +159,4 @@ impl XvcS3StorageOperations for XvcMinioStorage { fn region(&self) -> String { self.region.clone() } - - // async fn a_init( - // self, - // output: &XvcOutputSender, - // _xvc_root: &xvc_core::XvcRoot, - // ) -> Result<(XvcStorageInitEvent, Self)> { - // let bucket = self.get_bucket()?; - // let guid = self.guid.clone(); - // let guid_str = self.guid.to_string(); - // let guid_bytes = guid_str.as_bytes(); - // - // watch!(bucket); - // watch!(guid); - // watch!(guid_str); - // - // let res_response = bucket - // .put_object( - // format!("{}/{}", self.remote_prefix, XVC_STORAGE_GUID_FILENAME), - // guid_bytes, - // ) - // .await; - // - // match res_response { - // Ok(_) => Ok((XvcStorageInitEvent { guid }, self)), - // Err(err) => { - // error!(output, "{}", err); - // Err(Error::S3Error { source: err }) - // } - // } - // } - // - // async fn a_list( - // &self, - // output: &XvcOutputSender, - // xvc_root: &xvc_core::XvcRoot, - // ) -> Result { - // let credentials = self.credentials()?; - // let region = Region::from_str(&self.region).unwrap_or("us-east-1".parse().unwrap()); - // let bucket = Bucket::new(&self.bucket_name, region, credentials)?; - // let _guid = self.guid.clone(); - // let guid_str = self.guid.to_string(); - // let _guid_bytes = guid_str.as_bytes(); - // let xvc_guid = xvc_root.config().guid().unwrap(); - // let prefix = self.remote_prefix.clone(); - // - // let res_list = bucket - // .list( - // format!("{}/{}", self.remote_prefix, xvc_guid), - // Some("/".to_string()), - // ) - // .await; - // - // match res_list { - // Ok(list_all) => { - // // select only the matching elements - // let re = Regex::new(&format!( - // "{prefix}/{xvc_guid}/{cp}/{d3}/{d3}/{d58}/0\\..*$", - // cp = r#"[a-zA-Z][0-9]"#, - // d3 = r#"[0-9A-Fa-f]{3}"#, - // d58 = r#"[0-9A-Fa-f]{58}"# - // )) - // .unwrap(); - // - // let paths = list_all - // .iter() - // .filter_map(|e| { - // if re.is_match(e.name.as_ref()) { - // Some(XvcStoragePath::from_str(&e.name).unwrap()) - // } else { - // None - // } - // }) - // .collect(); - // - // Ok(XvcStorageListEvent { - // guid: self.guid.clone(), - // paths, - // }) - // } - // - // Err(err) => { - // error!(output, "{}", err); - // Err(Error::S3Error { source: err }) - // } - // } - // } \{ - // todo!() - // \} - // - // async fn a_send( - // &self, - // output: &XvcOutputSender, - // xvc_root: &xvc_core::XvcRoot, - // paths: &[xvc_core::XvcCachePath], - // _force: bool, - // ) -> crate::Result { - // let repo_guid = xvc_root - // .config() - // .guid() - // .ok_or_else(|| crate::Error::NoRepositoryGuidFound)?; - // let mut copied_paths = Vec::::new(); - // - // let bucket = self.get_bucket()?; - // - // for cache_path in paths { - // watch!(cache_path); - // let remote_path = self.build_remote_path(&repo_guid, cache_path); - // let abs_cache_path = cache_path.to_absolute_path(xvc_root); - // watch!(abs_cache_path); - // - // let mut path = tokio::fs::File::open(&abs_cache_path).await?; - // watch!(path); - // - // let res_response = bucket - // .put_object_stream(&mut path, remote_path.as_str()) - // .await; - // - // match res_response { - // Ok(_) => { - // info!(output, "{} -> {}", abs_cache_path, remote_path.as_str()); - // copied_paths.push(remote_path); - // watch!(copied_paths.len()); - // } - // Err(err) => { - // error!(output, "{}", err); - // } - // } - // } - // - // Ok(XvcStorageSendEvent { - // guid: self.guid.clone(), - // paths: copied_paths, - // }) - // } \{ - // todo!() - // \} - // - // async fn a_receive( - // &self, - // output: &XvcOutputSender, - // xvc_root: &xvc_core::XvcRoot, - // paths: &[xvc_core::XvcCachePath], - // _force: bool, - // ) -> Result<(XvcStorageTempDir, XvcStorageReceiveEvent)> { - // let repo_guid = xvc_root - // .config() - // .guid() - // .ok_or_else(|| crate::Error::NoRepositoryGuidFound)?; - // let mut copied_paths = Vec::::new(); - // - // let bucket = self.get_bucket()?; - // let temp_dir = XvcStorageTempDir::new()?; - // - // for cache_path in paths { - // watch!(cache_path); - // let remote_path = self.build_remote_path(&repo_guid, cache_path); - // let abs_cache_dir = temp_dir.temp_cache_dir(cache_path)?; - // fs::create_dir_all(&abs_cache_dir)?; - // let abs_cache_path = temp_dir.temp_cache_path(cache_path)?; - // watch!(abs_cache_path); - // - // let response_data_stream = bucket.get_object_stream(remote_path.as_str()).await; - // - // match response_data_stream { - // Ok(mut response) => { - // info!(output, "{} -> {}", remote_path.as_str(), abs_cache_path); - // let mut async_cache_path = tokio::fs::File::create(&abs_cache_path).await?; - // while let Some(chunk) = response.bytes().next().await { - // async_cache_path.write_all(&chunk).await?; - // } - // copied_paths.push(remote_path); - // watch!(copied_paths.len()); - // } - // Err(err) => { - // error!(output, "{}", err); - // } - // } - // } - // - // Ok(( - // temp_dir, - // XvcStorageReceiveEvent { - // guid: self.guid.clone(), - // paths: copied_paths, - // }, - // )) - // } \{ - // todo!() - // \} - // - // async fn a_delete( - // &self, - // output: &XvcOutputSender, - // xvc_root: &xvc_core::XvcRoot, - // paths: &[XvcCachePath], - // ) -> Result { - // let repo_guid = xvc_root - // .config() - // .guid() - // .ok_or_else(|| crate::Error::NoRepositoryGuidFound)?; - // let mut deleted_paths = Vec::::new(); - // - // let bucket = self.get_bucket()?; - // - // for cache_path in paths { - // watch!(cache_path); - // let remote_path = self.build_remote_path(&repo_guid, cache_path); - // bucket.delete_object(remote_path.as_str()).await?; - // info!(output, "[DELETE] {}", remote_path.as_str()); - // deleted_paths.push(remote_path); - // } - // - // Ok(XvcStorageDeleteEvent { - // guid: self.guid.clone(), - // paths: deleted_paths, - // }) - // } \{ - // todo!() - // \} } diff --git a/storage/src/storage/mod.rs b/storage/src/storage/mod.rs index 82fc059c6..235a8722c 100644 --- a/storage/src/storage/mod.rs +++ b/storage/src/storage/mod.rs @@ -32,7 +32,7 @@ pub use local::XvcLocalStorage; use serde::{Deserialize, Serialize}; use tempfile::TempDir; use uuid::Uuid; -use xvc_logging::{panic, watch, XvcOutputSender}; +use xvc_logging::{error, panic, watch, XvcOutputSender}; use xvc_walker::AbsolutePath; use crate::{Error, Result, StorageIdentifier}; @@ -73,7 +73,7 @@ pub enum XvcStorage { #[cfg(feature = "digital-ocean")] DigitalOcean(digital_ocean::XvcDigitalOceanStorage), } -persist!(XvcStorage, "remote"); +persist!(XvcStorage, "storage"); impl Display for XvcStorage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -108,28 +108,28 @@ impl Display for XvcStorage { ), #[cfg(feature = "s3")] - XvcStorage::S3(s3r) => write!( + XvcStorage::S3(r) => write!( f, "S3: {}\t{}\t{}.{}/{}", - s3r.name, s3r.guid, s3r.region, s3r.bucket_name, s3r.storage_prefix + r.name, r.guid, r.region, r.bucket_name, r.storage_prefix ), #[cfg(feature = "minio")] XvcStorage::Minio(mr) => write!( f, "Minio: {}\t{}\t{}.{}/{}", - mr.name, mr.guid, mr.endpoint, mr.bucket_name, mr.remote_prefix + mr.name, mr.guid, mr.endpoint, mr.bucket_name, mr.storage_prefix ), #[cfg(feature = "r2")] XvcStorage::R2(r2r) => write!( f, "R2: {}\t{}\t{} {}/{}", - r2r.name, r2r.guid, r2r.account_id, r2r.bucket_name, r2r.remote_prefix + r2r.name, r2r.guid, r2r.account_id, r2r.bucket_name, r2r.storage_prefix ), #[cfg(feature = "gcs")] XvcStorage::Gcs(gcsr) => write!( f, "GCS: {}\t{}\t{}.{}/{}", - gcsr.name, gcsr.guid, gcsr.region, gcsr.bucket_name, gcsr.remote_prefix + gcsr.name, gcsr.guid, gcsr.region, gcsr.bucket_name, gcsr.storage_prefix ), #[cfg(feature = "wasabi")] XvcStorage::Wasabi(wr) => write!( @@ -141,14 +141,14 @@ impl Display for XvcStorage { XvcStorage::DigitalOcean(dor) => write!( f, "DO: {}\t{}\t{}.{}/{}", - dor.name, dor.guid, dor.region, dor.bucket_name, dor.remote_prefix + dor.name, dor.guid, dor.region, dor.bucket_name, dor.storage_prefix ), } } } /// All storages implement this trait. xvc storage new and xvc file send / bring / remove -/// commands use this trait to communicate with the remotes. +/// commands use this trait to communicate with the storages. pub trait XvcStorageOperations { /// The init operation is creates a directory with the "short guid" of the Xvc repository and /// adds a .xvc-guid file with the guid of the storage. @@ -375,7 +375,7 @@ impl XvcStorageTempDir { /// It uses [RelativePathBuf] internally. #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize, Display)] pub struct XvcStoragePath(RelativePathBuf); -persist!(XvcStoragePath, "remote-path"); +persist!(XvcStoragePath, "storage-path"); impl From for XvcStoragePath { fn from(p: String) -> Self { @@ -397,7 +397,7 @@ impl AsRef for XvcStoragePath { } impl XvcStoragePath { - /// The remote path of a cache path is like {guid}/{cache-path} + /// The storage path of a cache path is like {guid}/{cache-path} /// ⚠️ The separator between {guid} and {cache-path} is always / pub fn new(xvc_root: &XvcRoot, local: &XvcCachePath) -> Self { let guid = xvc_root.config().guid().unwrap(); @@ -457,7 +457,7 @@ pub fn get_storage_record( identifier: &StorageIdentifier, ) -> Result { let store: XvcStore = xvc_root.load_store()?; - let remote_store = store.filter(|_, r| match identifier { + let storage_store = store.filter(|_, r| match identifier { StorageIdentifier::Name(ref n) => match r { XvcStorage::Local(r) => r.name == *n, XvcStorage::Generic(r) => r.name == *n, @@ -494,18 +494,18 @@ pub fn get_storage_record( }, }); - if remote_store.is_empty() { - panic!(output_snd, "Cannot find remote {}", identifier); + if storage_store.is_empty() { + error!(output_snd, "Cannot find remote {}", identifier); } - if remote_store.len() > 1 { - panic!(output_snd, "Ambiguous remote identifier: {}", identifier); + if storage_store.len() > 1 { + error!(output_snd, "Ambiguous remote identifier: {} Please use Storage GUID.", identifier); } - let (_, remote) = - remote_store + let (_, storage) = + storage_store .first() - .ok_or_else(|| Error::CannotFindRemoteWithIdentifier { + .ok_or_else(|| Error::CannotFindStorageWithIdentifier { identifier: identifier.clone(), })?; - Ok(remote.clone()) + Ok(storage.clone()) } diff --git a/storage/src/storage/r2.rs b/storage/src/storage/r2.rs index dc6830423..abdef9a4b 100644 --- a/storage/src/storage/r2.rs +++ b/storage/src/storage/r2.rs @@ -19,7 +19,7 @@ use super::{XvcStorageListEvent, XvcStoragePath}; /// Configure a new Cloudflare R2 remote storage. /// -/// `account_id`, `bucket_name`, and `remote_prefix` sets a URL for the +/// `account_id`, `bucket_name`, and `storage_prefix` sets a URL for the /// storage location. /// /// This creates a [XvcR2Storage], calls its @@ -32,19 +32,19 @@ pub fn cmd_new_r2( name: String, account_id: String, bucket_name: String, - remote_prefix: String, + storage_prefix: String, ) -> Result<()> { - let mut remote = XvcR2Storage { + let mut storage = XvcR2Storage { guid: XvcStorageGuid::new(), name, account_id, bucket_name, - remote_prefix, + storage_prefix, }; - watch!(remote); + watch!(storage); - let init_event = remote.init(output_snd, xvc_root)?; + let init_event = storage.init(output_snd, xvc_root)?; watch!(init_event); xvc_root.with_r1nstore_mut(|store: &mut R1NStore| { @@ -52,7 +52,7 @@ pub fn cmd_new_r2( let event_e = xvc_root.new_entity(); store.insert( store_e, - XvcStorage::R2(remote.clone()), + XvcStorage::R2(storage.clone()), event_e, XvcStorageEvent::Init(init_event.clone()), ); @@ -73,12 +73,12 @@ pub struct XvcR2Storage { pub account_id: String, /// Bucket name of the R2 storage pub bucket_name: String, - /// Remote prefix in the bucket - pub remote_prefix: String, + /// Remote path prefix in the bucket + pub storage_prefix: String, } impl XvcR2Storage { - fn remote_specific_credentials(&self) -> Result { + fn storage_specific_credentials(&self) -> Result { Credentials::new( Some(&env::var(format!( "XVC_STORAGE_ACCESS_KEY_ID_{}", @@ -105,8 +105,8 @@ impl XvcR2Storage { } impl XvcS3StorageOperations for XvcR2Storage { - fn remote_prefix(&self) -> String { - self.remote_prefix.clone() + fn storage_prefix(&self) -> String { + self.storage_prefix.clone() } fn guid(&self) -> &XvcStorageGuid { @@ -115,10 +115,10 @@ impl XvcS3StorageOperations for XvcR2Storage { fn bucket_name(&self) -> String { self.bucket_name.clone() } - fn build_remote_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { + fn build_storage_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { XvcStoragePath::from(format!( "{}/{}/{}", - self.remote_prefix, + self.storage_prefix, self.guid(), cache_path )) @@ -130,7 +130,7 @@ impl XvcS3StorageOperations for XvcR2Storage { } fn credentials(&self) -> Result { - match self.remote_specific_credentials() { + match self.storage_specific_credentials() { Ok(c) => Ok(c), Err(e1) => match self.storage_type_credentials() { Ok(c) => Ok(c), @@ -161,11 +161,11 @@ impl XvcS3StorageOperations for XvcR2Storage { ) -> Result { let bucket = self.get_bucket()?; let xvc_guid = xvc_root.config().guid().unwrap(); - let prefix = self.remote_prefix.clone(); + let prefix = self.storage_prefix.clone(); let res_list = bucket .list( - format!("{}/{}", self.remote_prefix, xvc_guid), + format!("{}/{}", self.storage_prefix, xvc_guid), Some("/".to_string()), ) .await; diff --git a/storage/src/storage/rsync.rs b/storage/src/storage/rsync.rs index 24e242cbb..aaa054440 100644 --- a/storage/src/storage/rsync.rs +++ b/storage/src/storage/rsync.rs @@ -60,7 +60,7 @@ pub fn cmd_new_rsync( Ok(()) } -/// Specifies an Rsync remote +/// Specifies an Rsync remote storage #[derive(Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Serialize, Deserialize)] pub struct XvcRsyncStorage { /// The GUID of the storage @@ -237,8 +237,8 @@ impl XvcRsyncStorage { xvc_guid: &str, cache_path: &XvcCachePath, ) -> Result { - let remote_dir = self.ssh_cache_dir(xvc_guid, cache_path); - self.ssh_cmd(ssh_executable, &format!("mkdir -p '{}'", remote_dir)) + let storage_dir = self.ssh_cache_dir(xvc_guid, cache_path); + self.ssh_cmd(ssh_executable, &format!("mkdir -p '{}'", storage_dir)) } } @@ -265,18 +265,18 @@ impl XvcStorageOperations for XvcRsyncStorage { fs::write(&local_guid_path, format!("{}", self.guid))?; watch!(local_guid_path); - let remote_guid_path = self.rsync_path_url(XVC_STORAGE_GUID_FILENAME); + let storage_guid_path = self.rsync_path_url(XVC_STORAGE_GUID_FILENAME); - watch!(remote_guid_path); + watch!(storage_guid_path); let rsync_result = - self.rsync_copy_to_storage(&rsync_executable, &local_guid_path, &remote_guid_path)?; + self.rsync_copy_to_storage(&rsync_executable, &local_guid_path, &storage_guid_path)?; watch!(rsync_result); info!( output, "Initialized:\n{}\n{}\n", - remote_guid_path, + storage_guid_path, rsync_result.stdout_str() ); @@ -343,11 +343,11 @@ impl XvcStorageOperations for XvcRsyncStorage { let mut storage_paths = Vec::::with_capacity(paths.len()); paths.iter().for_each(|cache_path| { let local_path = cache_path.to_absolute_path(xvc_root); - let remote_url = self.rsync_cache_url(&xvc_guid, cache_path); - let _remote_dir_res = self.create_storage_dir(&ssh_executable, &xvc_guid, cache_path); + let storage_url = self.rsync_cache_url(&xvc_guid, cache_path); + uwr!(self.create_storage_dir(&ssh_executable, &xvc_guid, cache_path), output); // TODO: Handle possible error. let cmd_output = - self.rsync_copy_to_storage(&rsync_executable, &local_path, &remote_url); + self.rsync_copy_to_storage(&rsync_executable, &local_path, &storage_url); match cmd_output { Ok(cmd_output) => { diff --git a/storage/src/storage/s3.rs b/storage/src/storage/s3.rs index 4e32b86fb..3d3d893df 100644 --- a/storage/src/storage/s3.rs +++ b/storage/src/storage/s3.rs @@ -19,7 +19,7 @@ use super::XvcStoragePath; /// Configure a new Amazon Web Services S3 remote storage. /// -/// `bucket_name`, `region` and `remote_prefix` sets a URL for the storage +/// `bucket_name`, `region` and `storage_prefix` sets a URL for the storage /// location. /// /// This creates a [XvcS3Storage], calls its @@ -33,7 +33,7 @@ pub fn cmd_new_s3( bucket_name: String, storage_prefix: String, ) -> Result<()> { - let mut remote = XvcS3Storage { + let mut storage = XvcS3Storage { guid: XvcStorageGuid::new(), name, region, @@ -41,9 +41,9 @@ pub fn cmd_new_s3( storage_prefix, }; - watch!(remote); + watch!(storage); - let init_event = remote.init(output_snd, xvc_root)?; + let init_event = storage.init(output_snd, xvc_root)?; watch!(init_event); xvc_root.with_r1nstore_mut(|store: &mut R1NStore| { @@ -51,7 +51,7 @@ pub fn cmd_new_s3( let event_e = xvc_root.new_entity(); store.insert( store_e, - XvcStorage::S3(remote.clone()), + XvcStorage::S3(storage.clone()), event_e, XvcStorageEvent::Init(init_event.clone()), ); @@ -70,7 +70,7 @@ pub struct XvcS3Storage { /// `bucket_name.region.s3.amazonaws.com/storage_prefix/.xvc-guid` to identify the /// remote location. pub guid: XvcStorageGuid, - /// Name of the remote to be used in commands. + /// Name of the remote storage to be used in commands. /// /// It doesn't have to be unique, though in practice setting unique names is /// preferred. @@ -89,7 +89,7 @@ pub struct XvcS3Storage { } impl XvcS3Storage { - fn remote_specific_credentials(&self) -> Result { + fn storage_specific_credentials(&self) -> Result { Credentials::new( Some(&env::var(format!( "XVC_STORAGE_ACCESS_KEY_ID_{}", @@ -116,7 +116,7 @@ impl XvcS3Storage { } impl XvcS3StorageOperations for XvcS3Storage { - fn remote_prefix(&self) -> String { + fn storage_prefix(&self) -> String { self.storage_prefix.clone() } @@ -133,7 +133,7 @@ impl XvcS3StorageOperations for XvcS3Storage { } fn credentials(&self) -> Result { - match self.remote_specific_credentials() { + match self.storage_specific_credentials() { Ok(c) => Ok(c), Err(e1) => match self.storage_type_credentials() { Ok(c) => Ok(c), @@ -151,13 +151,13 @@ impl XvcS3StorageOperations for XvcS3Storage { self.bucket_name.clone() } - async fn write_remote_guid(&self) -> Result<()> { + async fn write_storage_guid(&self) -> Result<()> { let guid_str = self.guid().to_string(); let guid_bytes = guid_str.as_bytes(); let bucket = self.get_bucket()?; let response = bucket .put_object( - format!("{}/{}", self.remote_prefix(), XVC_STORAGE_GUID_FILENAME), + format!("{}/{}", self.storage_prefix(), XVC_STORAGE_GUID_FILENAME), guid_bytes, ) .await; @@ -168,11 +168,11 @@ impl XvcS3StorageOperations for XvcS3Storage { } } - fn build_remote_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { + fn build_storage_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { XvcStoragePath::from(format!( "{}/{}/{}/{}", self.bucket_name(), - self.remote_prefix(), + self.storage_prefix(), self.guid(), cache_path )) diff --git a/storage/src/storage/wasabi.rs b/storage/src/storage/wasabi.rs index 19b38f372..639d18cb7 100644 --- a/storage/src/storage/wasabi.rs +++ b/storage/src/storage/wasabi.rs @@ -16,7 +16,7 @@ use anyhow::anyhow; use super::async_common::XvcS3StorageOperations; use super::XvcStoragePath; -/// Configure a new Wasabi storage remote. +/// Configure a new Wasabi remote storage. /// /// `bucket_name`, `endpoint` and `storage_prefix` sets a URL for the storage /// location. @@ -32,7 +32,7 @@ pub(crate) fn cmd_new_wasabi( endpoint: String, storage_prefix: String, ) -> Result<()> { - let mut remote = XvcWasabiStorage { + let mut storage = XvcWasabiStorage { guid: XvcStorageGuid::new(), name, endpoint, @@ -40,9 +40,9 @@ pub(crate) fn cmd_new_wasabi( storage_prefix, }; - watch!(remote); + watch!(storage); - let init_event = remote.init(output_snd, xvc_root)?; + let init_event = storage.init(output_snd, xvc_root)?; watch!(init_event); xvc_root.with_r1nstore_mut(|store: &mut R1NStore| { @@ -50,7 +50,7 @@ pub(crate) fn cmd_new_wasabi( let event_e = xvc_root.new_entity(); store.insert( store_e, - XvcStorage::Wasabi(remote.clone()), + XvcStorage::Wasabi(storage.clone()), event_e, XvcStorageEvent::Init(init_event.clone()), ); @@ -91,7 +91,7 @@ pub struct XvcWasabiStorage { } impl XvcWasabiStorage { - fn remote_specific_credentials(&self) -> Result { + fn storage_specific_credentials(&self) -> Result { Credentials::new( Some(&env::var(format!( "XVC_STORAGE_ACCESS_KEY_ID_{}", @@ -118,7 +118,7 @@ impl XvcWasabiStorage { } impl XvcS3StorageOperations for XvcWasabiStorage { - fn remote_prefix(&self) -> String { + fn storage_prefix(&self) -> String { self.storage_prefix.clone() } @@ -147,7 +147,7 @@ impl XvcS3StorageOperations for XvcWasabiStorage { } fn credentials(&self) -> Result { - match self.remote_specific_credentials() { + match self.storage_specific_credentials() { Ok(c) => Ok(c), Err(e1) => match self.storage_type_credentials() { Ok(c) => Ok(c), @@ -161,7 +161,7 @@ impl XvcS3StorageOperations for XvcWasabiStorage { .map_err(|e| e.into()) } - fn build_remote_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { + fn build_storage_path(&self, cache_path: &XvcCachePath) -> XvcStoragePath { XvcStoragePath::from(format!( "{}/{}/{}", self.storage_prefix, diff --git a/test_helper/Cargo.toml b/test_helper/Cargo.toml index 0062e7fa4..fa20c7380 100644 --- a/test_helper/Cargo.toml +++ b/test_helper/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-test-helper" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Unit test helper functions for Xvc" authors = ["Emre Şahin "] @@ -23,7 +23,7 @@ debug = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -xvc-logging = { version = "0.6.7", path = "../logging/" } +xvc-logging = { version = "0.6.8", path = "../logging/" } rand = "^0.8" log = "^0.4" diff --git a/walker/Cargo.toml b/walker/Cargo.toml index 5cbcc717c..6c3296666 100644 --- a/walker/Cargo.toml +++ b/walker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-walker" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Xvc parallel file system walker with ignore features" authors = ["Emre Şahin "] @@ -19,7 +19,7 @@ crate-type = ["rlib"] debug = true [dependencies] -xvc-logging = { version = "0.6.7", path = "../logging" } +xvc-logging = { version = "0.6.8", path = "../logging" } globset = "^0.4" ## Parallelization @@ -41,7 +41,7 @@ itertools = "^0.12" regex = "^1.10" [dev-dependencies] -xvc-test-helper = { path = "../test_helper/", version = "0.6.7" } +xvc-test-helper = { path = "../test_helper/", version = "0.6.8" } test-case = "^3.3" [package.metadata.cargo-udeps.ignore] diff --git a/workflow_tests/Cargo.toml b/workflow_tests/Cargo.toml index 6d0ea06fe..5c3d3e8da 100644 --- a/workflow_tests/Cargo.toml +++ b/workflow_tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xvc-workflow-tests" -version = "0.6.7" +version = "0.6.8" edition = "2021" description = "Integration test suite for Xvc" authors = ["Emre Şahin "] @@ -23,15 +23,15 @@ debug = true [dependencies] -xvc = { version = "0.6.7", path = "../lib" } -xvc-config = { version = "0.6.7", path = "../config" } -xvc-core = { version = "0.6.7", path = "../core" } -xvc-logging = { version = "0.6.7", path = "../logging" } -xvc-ecs = { version = "0.6.7", path = "../ecs" } -xvc-file = { version = "0.6.7", path = "../file" } -xvc-pipeline = { version = "0.6.7", path = "../pipeline" } -xvc-walker = { version = "0.6.7", path = "../walker" } -xvc-storage = { version = "0.6.7", path = "../storage" } +xvc = { version = "0.6.8", path = "../lib" } +xvc-config = { version = "0.6.8", path = "../config" } +xvc-core = { version = "0.6.8", path = "../core" } +xvc-logging = { version = "0.6.8", path = "../logging" } +xvc-ecs = { version = "0.6.8", path = "../ecs" } +xvc-file = { version = "0.6.8", path = "../file" } +xvc-pipeline = { version = "0.6.8", path = "../pipeline" } +xvc-walker = { version = "0.6.8", path = "../walker" } +xvc-storage = { version = "0.6.8", path = "../storage" } ## packages for testing clap = { version = "^4.4", features = ["derive"] } @@ -70,7 +70,7 @@ proptest = "^1.4" test-case = "^3.3" globset = "^0.4" escargot = "^0.5" -xvc-test-helper = { version = "0.6.7", path = "../test_helper" } +xvc-test-helper = { version = "0.6.8", path = "../test_helper" } shellfn = "^0.1" jwalk = "^0.8" anyhow = "^1.0" diff --git a/workflow_tests/src/lib.rs b/workflow_tests/src/lib.rs index 9b05d9691..94fa246fb 100755 --- a/workflow_tests/src/lib.rs +++ b/workflow_tests/src/lib.rs @@ -9,11 +9,11 @@ pub use xvc_pipeline as pipeline; pub use xvc_logging::watch; -use xvc::{cli::XvcCLI, error::Result}; +use xvc::{cli::XvcCLI, error::Result, XvcRootOpt}; /// Run xvc in another process with `args` in `xvc_root_opt`. /// /// See [xvc::cli::test_dispatch]. -pub fn dispatch(cli_opts: XvcCLI) -> Result<()> { +pub fn dispatch(cli_opts: XvcCLI) -> Result { xvc::cli::dispatch(cli_opts) } diff --git a/workflow_tests/src/main.rs b/workflow_tests/src/main.rs index be0e58400..ff4c0cd24 100755 --- a/workflow_tests/src/main.rs +++ b/workflow_tests/src/main.rs @@ -9,5 +9,6 @@ use xvc::error::Result; /// fn main() -> Result<()> { let cli_opts = xvc::cli::XvcCLI::from_args_os(std::env::args_os())?; - xvc::cli::dispatch(cli_opts) + xvc::cli::dispatch(cli_opts)?; + Ok(()) } diff --git a/workflow_tests/tests/common/mod.rs b/workflow_tests/tests/common/mod.rs index 25a368d1f..7fc9003c0 100644 --- a/workflow_tests/tests/common/mod.rs +++ b/workflow_tests/tests/common/mod.rs @@ -152,7 +152,7 @@ pub fn run_in_example_xvc(with_git: bool) -> Result { } } -/// Create a temporary XVC directory that's also Git repository +/// Create a temporary Xvc directory that's also Git repository pub fn run_in_temp_xvc_dir() -> Result { let the_dir = run_in_temp_git_dir(); watch!(&the_dir); diff --git a/workflow_tests/tests/test_init_1.rs b/workflow_tests/tests/test_init_1.rs index 49d652fd8..b74575768 100644 --- a/workflow_tests/tests/test_init_1.rs +++ b/workflow_tests/tests/test_init_1.rs @@ -8,7 +8,7 @@ use xvc::init::InitCLI; use xvc::error::Result; #[test] -fn test_remote_add() -> Result<()> { +fn test_storage_add() -> Result<()> { common::test_logging(LevelFilter::Trace); let _the_dir = common::run_in_temp_dir(); let xvc_root = xvc::init::run( diff --git a/workflow_tests/tests/test_pipeline_dag.rs b/workflow_tests/tests/test_pipeline_dag.rs index ae57403ab..d15ed39bd 100644 --- a/workflow_tests/tests/test_pipeline_dag.rs +++ b/workflow_tests/tests/test_pipeline_dag.rs @@ -68,6 +68,7 @@ fn test_pipeline_dag() -> Result<()> { xsd("glob_dep", "--glob", "*.txt")?; xsd("glob_dep", "--glob-items", "data/*")?; xsd("glob_dep", "--param", "params.yaml:model.conv_units")?; + // FIXME: There is an error here regarding regex format xsd("glob_dep", "--regex", "requirements.txt:^tensorflow")?; xsom("glob_dep", "glob_dep.json")?; xsof("glob_dep", "def.txt")?; diff --git a/workflow_tests/tests/test_storage_new_s3.rs b/workflow_tests/tests/test_storage_new_s3.rs index 7a14d64dd..d42725b0c 100644 --- a/workflow_tests/tests/test_storage_new_s3.rs +++ b/workflow_tests/tests/test_storage_new_s3.rs @@ -252,7 +252,7 @@ fn test_storage_new_s3() -> Result<()> { assert!(n_storage_files_after == n_local_files_after_pull); assert!(PathBuf::from(the_file).exists()); - // Set remote specific passwords and remove AWS ones + // Set storage specific passwords and remove AWS ones env::set_var("XVC_STORAGE_ACCESS_KEY_ID_s3-storage", access_key); env::set_var("XVC_STORAGE_SECRET_KEY_s3-storage", secret_key);