Fix xvc file track memory issue (#260)

.github/workflows/publish.yml

@@ -22,4 +22,4 @@ jobs:
       - name: Publish to crates.io
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
-        run: for p in xvc-logging xvc-test-helper xvc-walker xvc-ecs  xvc-config xvc-core xvc-storage xvc-file xvc-pipelines xvc ; do cargo publish --package $p ; done
+        run: for p in xvc-logging xvc-test-helper xvc-walker xvc-ecs xvc-config xvc-core xvc-storage xvc-file xvc-pipelines xvc ; do cargo publish --package $p ; done

.github/workflows/rust.yml

@@ -28,7 +28,7 @@ jobs:
           - build: nightly
             rust: nightly
             # rust: nightly-2024-01-01
-            test-args: --features test-ci --no-fail-fast
+            test-args: --features test-ci # --no-fail-fast
             ## for submitters other than me, I'll add another job here.
             # test-args: --no-fail-fast
             # test-args: --all-features
@@ -67,6 +67,7 @@ jobs:
           brew install tree
           brew install lsd
           brew install python3
+          brew install minio/stable/mc
 
       - name: Git config for automated Git tests
         run: git config --global user.name 'Xvc Rabbit' && git config --global user.email '[email protected]' && git config --global init.defaultBranch main
@@ -105,6 +106,9 @@ jobs:
       - name: Check if xvc is in PATH
         run: tree $GITHUB_WORKSPACE && xvc --help
 
+      - name: Run Current Dev Tests
+        run: $GITHUB_WORKSPACE/run-tests.zsh
+
       - name: Test
         if: matrix.coverage
         uses: actions-rs/cargo@v1

CHANGELOG.md

@@ -1,5 +1,19 @@
+# Xvc Changelog
+
 ## Unreleased
 
+## 0.6.11 (2024-09-04)
+
+- Bump dependencies
+- Replace globset with fast-glob for memory usage
+- Remove --details option from xvc check-ignore
+- Fixed xvc check-ignore to work with supplied paths
+- Fixed loading store targets from xvc file list
+- Directory targets in various commands doesn't require / at the end when they only exist in the cache
+- Removed some duplicate tests from ignore
+- Minio tests now use mc instead of s3cmd
+- Add a step to run a subset of tests in CI for faster feedback
+
 ## 0.6.10 (2024-08-04)
 
 - PR: <https://github.com/iesahin/xvc/pull/259>

book/src/ref/xvc-check-ignore.md

@@ -4,7 +4,7 @@
 
 Check whether a path is ignored or whitelisted by Xvc.
 
-## Synopsis 
+## Synopsis
 
 ```console
 $ xvc check-ignore --help
@@ -17,53 +17,81 @@ Arguments:
           Targets to check. If no targets are provided, they are read from stdin
 
 Options:
-  -d, --details
-          Show the exclude patterns along with each target path. A series of lines are printed in this format: <path/to/.xvcignore>:<line_num>:<pattern> <target_path>
-
       --ignore-filename <IGNORE_FILENAME>
           Filename that contains ignore rules
           
           This can be set to .gitignore to test whether Git and Xvc work the same way.
           
           [default: .xvcignore]
 
-  -n, --non-matching
-          Include the target paths which don’t match any pattern in the --details list. All fields in each line, except for <target_path>, will be empty. Has no effect without --details
-
   -h, --help
           Print help (see a summary with '-h')
 
 ```
 
 ## Examples
 
+```console
+$ git init
+...
+$ xvc init
+```
+
+You can add files and directories to be ignored by Xvc to `.xvcignore` files.
+
+```console
+$ zsh -cl "echo 'my-dir/my-file' >> .xvcignore"
+```
+
 By default it checks the files supplied from `stdin`.
 
-```shell
-$ xvc check-ignore
+```console
+$ zsh -cl 'echo my-dir/my-file | xvc check-ignore'
+[IGNORE] [CWD]/my-dir/my-file
+
+```
+
+The `.xvcignore` file format is identical to [`.gitignore` file format](https://git-scm.com/docs/gitignore).
+
+```console
+$ cat .xvcignore
+
+# Add patterns of files xvc should ignore, which could improve
+# the performance.
+# It's in the same format as .gitignore files.
+
+.DS_Store
 my-dir/my-file
+
 ```
 
-If you supply paths from the CLI, they are checked instead. 
+If you supply paths from the CLI, they are checked against the ignore rules in `.xvcignore`.
 
-```shell
+```console
 $ xvc check-ignore my-dir/my-file another-dir/another-file
+[IGNORE] [CWD]/my-dir/my-file
+[NO MATCH] [CWD]/another-dir/another-file
+
 ```
 
-If you're looking which `.xvcignore` file ignores (or whitelists) a certain path, you can use `--details`.
+You can also add whitelist patterns to `,.xvcignore` files.
 
-```shell
-$ xvc check-ignore --details my-dir/my-file another-dir/another-file
+```console
+$ zsh -cl "echo '!another-dir/*' >> .xvcignore"
 ```
 
-`.xvcignore` file format is identical to [`.gitignore` file format](https://git-scm.com/docs/gitignore).
-This utility can be used to check any other ignore rules in other files as well.
-You can specify an alternative ignore filename with `--ignore-filename` option.
-The below command is identical to `git check-ignore` and should give the same results.
+```console
+$ xvc check-ignore my-dir/my-file another-dir/another-file
+[IGNORE] [CWD]/my-dir/my-file
+[WHITELIST] [CWD]/another-dir/another-file
 
-```shell
-$ xvc check-ignore --ignore-filename .gitignore 
 ```
 
+This utility can be used to check any other ignore rules in other files as well.
+You can specify an alternative ignore filename with `--ignore-filename` option.
+The below command is identical to `git check-ignore` and should give the same results.
 
+```console
+$ xvc check-ignore --ignore-filename .gitignore
 
+```

book/src/ref/xvc-file-list.md

@@ -168,7 +168,7 @@ Total #: 30 Workspace Size:       51195 Cached Size:           0
 
 ```
 
-By default the command hides dotfiles. If you also want to show them, you can use `--show-dot-files`/`-a` flag. 
+By default the command hides dotfiles. If you also want to show them, you can use `--show-dot-files`/`-a` flag.
 
 ```console
 $ xvc file list --sort name-asc --show-dot-files
@@ -209,7 +209,7 @@ Total #: 32 Workspace Size:       51443 Cached Size:           0
 
 ```
 
-You can also hide the summary below the list to get only the list of files. 
+You can also hide the summary below the list to get only the list of files.
 
 ```console
 $ xvc file list  --sort name-asc --no-summary
@@ -276,7 +276,7 @@ If you add another set of files as hardlinks to the cached copies, it will
 print the second letter as `H`.
 
 ```console
-$ xvc file track dir-0002 --recheck-method hardlink
+$ xvc file track dir-0002/ --recheck-method hardlink
 
 $ xvc file list dir-0002
 FH        2005 [..] 447933dc 447933dc dir-0002/file-0005.bin

book/src/ref/xvc-storage-new-local.md

@@ -117,7 +117,7 @@ $ xvc file remove --from-storage backup dir-0001/
 
 `--name NAME` is not checked to be unique but you should use unique storage names to refer them later.
 
-`--path PATH`  should be accessible for writing and shouldn't already exist.
+`--path PATH` should be accessible for writing and shouldn't already exist.
 
 ## Technical Details
 

book/src/ref/xvc-storage-new-minio.md

@@ -78,7 +78,7 @@ $ xvc file track dir-0001
 You can define a storage bucket as storage and begin to use it.
 
 ```console,ignore
-$ xvc storage new minio --name backup --endpoint http://emresult.com:9000 --bucket-name one --region us-east-1 --storage-prefix xvc
+$ xvc storage new minio --name backup --endpoint http://e1.xvc.dev:9000 --bucket-name xvc-tests --region us-east-1 --storage-prefix xvc
 
 ```
 
@@ -147,7 +147,6 @@ You may need to consider this when you have servers running in exact URLs.
 If you have a `http://minio.example.com:9001` as a Minio server, you may want to supply `http://example.com:9001` as the endpoint, and `minio` as the bucket name to form the correct URL.
 This behavior may change in the future.
 
-
 ## Technical Details
 
 This command requires Xvc to be compiled with `minio` feature, which is _on_ by default.
@@ -162,5 +161,3 @@ A file that's found in `.xvc/{{HASH_PREFIX}}/{{CACHE_PATH}}` is saved to `http:/
 `{{REPO_ID}}` is the unique identifier for the repository created during `xvc init`.
 Hence if you use a common storage for different Xvc projects, their files are kept under different directories.
 There is no inter-project deduplication.
-
-

book/src/ref/xvc-storage-new-rsync.md

@@ -69,7 +69,7 @@ $ xvc file track dir-0001
 You can define a storage bucket as storage and begin to use it.
 
 ```console
-$ xvc storage new rsync --name backup --host one.emresult.com --user iex --storage-dir /tmp/xvc-backup/
+$ xvc storage new rsync --name backup --host e1.xvc.dev --user iex --storage-dir /tmp/xvc-backup/
 
 ```
 

config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xvc-config"
-version = "0.6.10"
+version = "0.6.11"
 edition = "2021"
 description = "Xvc configuration management"
 authors = ["Emre Şahin <[email protected]>"]
@@ -16,8 +16,8 @@ name = "xvc_config"
 crate-type = ["rlib"]
 
 [dependencies]
-xvc-logging = { version = "0.6.10", path = "../logging" }
-xvc-walker = { version = "0.6.10", path = "../walker" }
+xvc-logging = { version = "0.6.11", path = "../logging" }
+xvc-walker = { version = "0.6.11", path = "../walker" }
 
 
 ## Cli and config
@@ -33,7 +33,7 @@ crossbeam-channel = "^0.5"
 crossbeam = "^0.8"
 
 ## File system
-walkdir = "^2.4"
+walkdir = "^2.5"
 
 ## Logging and errors
 thiserror = "^1.0"
@@ -45,4 +45,4 @@ fern = { version = "^0.6", features = ["colored"] }
 regex = "^1.10"
 strum = "^0.26"
 strum_macros = "^0.26"
-lazy_static = "^1.4"
+lazy_static = "^1.5"

core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xvc-core"
-version = "0.6.10"
+version = "0.6.11"
 edition = "2021"
 description = "Xvc core for common elements for all commands"
 authors = ["Emre Şahin <[email protected]>"]
@@ -16,13 +16,13 @@ name = "xvc_core"
 crate-type = ["rlib"]
 
 [dependencies]
-xvc-config = { version = "0.6.10", path = "../config" }
-xvc-logging = { version = "0.6.10", path = "../logging" }
-xvc-ecs = { version = "0.6.10", path = "../ecs" }
-xvc-walker = { version = "0.6.10", path = "../walker" }
+xvc-config = { version = "0.6.11", path = "../config" }
+xvc-logging = { version = "0.6.11", path = "../logging" }
+xvc-ecs = { version = "0.6.11", path = "../ecs" }
+xvc-walker = { version = "0.6.11", path = "../walker" }
 
 ## Cli and config
-clap = { version = "^4.4", features = ["derive"] }
+clap = { version = "^4.5", features = ["derive"] }
 directories-next = "2.0"
 
 ## Hashing
@@ -38,20 +38,20 @@ serde = { version = "^1.0", features = ["derive"] }
 serde_yaml = "^0.9"
 serde_json = "^1.0"
 rmp = "^0.8"
-rmp-serde = "1.1.2"
+rmp-serde = "1.3.0"
 toml = "^0.8"
 
 ## Network
 reqwest = { version = "^0.11", features = ["blocking", "json", "gzip"] }
 
 ## Parallelization
-rayon = "^1.8"
+rayon = "^1.10"
 crossbeam-channel = "^0.5"
 crossbeam = "^0.8"
 
 ## File system
 jwalk = "^0.8"
-walkdir = "^2.4"
+walkdir = "^2.5"
 relative-path = { version = "^1.9", features = ["serde"] }
 path-absolutize = "^3.1"
 glob = "^0.3"
@@ -60,6 +60,7 @@ glob = "^0.3"
 thiserror = "^1.0"
 anyhow = "^1.0"
 log = "^0.4"
+peak_alloc = "^0.2"
 ### meta-logging-in-format is required for sled: https://github.com/spacejam/sled/issues/1384
 fern = { version = "^0.6", features = ["colored"] }
 
@@ -71,15 +72,15 @@ paste = "1.0"
 regex = "^1.10"
 strum = "^0.26"
 strum_macros = "^0.26"
-lazy_static = "^1.4"
-uuid = { version = "^1.6", features = ["serde", "v4", "fast-rng"] }
+lazy_static = "^1.5"
+uuid = { version = "^1.10", features = ["serde", "v4", "fast-rng"] }
 hex = { version = "^0.4", features = ["serde"] }
 cached = "^0.53"
 derive_more = "^0.99"
 itertools = "^0.13"
 
 
 [dev-dependencies]
-xvc-test-helper = { version = "0.6.10", path = "../test_helper/" }
-proptest = "^1.4"
+xvc-test-helper = { version = "0.6.11", path = "../test_helper/" }
+proptest = "^1.5"
 test-case = "^3.3"