Fix optional dependencies and cached globs (#259)

iesahin · Aug 4, 2024 · 59a7127 · 59a7127
1 parent eaabfbf
commit 59a7127
Show file tree

Hide file tree

Showing 38 changed files with 296 additions and 227 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -0,0 +1,25 @@
+name: Publish Crates
+
+on:
+  push:
+    tags:
+      - "v*.*.*" # Matches tags like v0.6.10 but also v0.6.10-alpha.5
+
+jobs:
+  publish:
+    # Don't publish alpha releases
+    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Publish to crates.io
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: for p in xvc-logging xvc-test-helper xvc-walker xvc-ecs  xvc-config xvc-core xvc-storage xvc-file xvc-pipelines xvc ; do cargo publish --package $p ; done
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -26,7 +26,8 @@ jobs:
           # - build: beta
           #   rust: beta
           - build: nightly
-            rust: nightly-2024-01-01
+            rust: nightly
+            # rust: nightly-2024-01-01
             test-args: --features test-ci --no-fail-fast
             ## for submitters other than me, I'll add another job here.
             # test-args: --no-fail-fast
@@ -158,7 +159,7 @@ jobs:
         with:
           use-cross: false
           command: build
-          args: --release --target ${{ matrix.target }}
+          args: --release --target ${{ matrix.target }} --features=bundled-openssl
 
       - name: Package
         shell: bash
@@ -199,7 +200,7 @@ jobs:
         with:
           use-cross: false
           command: build
-          args: --release --target ${{ matrix.target }}
+          args: --release --target ${{ matrix.target }} --features bundled-sqlite
 
       - name: Package
         shell: bash

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,18 +1,22 @@
-# Introduction
+## Unreleased
 
-This document is a change log that I write for the project as I develop. It's a
-tree, and subtasks are marked with indentation.
+## 0.6.10 (2024-08-04)
 
-## Unreleased
+- PR: <https://github.com/iesahin/xvc/pull/259>
+- Removed caching for globs that caused bugs in Python bindings and long running processes.
+- Documentation updates
+- Fix optional features. Now inter-workspace dependencies are defined by `default-features = false` on Cargo.toml
+- Added `bundled-openssl` feature to use `vendored` feature of `openssl` crate optionally. This is turned on for Windows builds on GA.
+- Crates are published from Github Actions
 
 ## 0.6.9 (2024-07-29)
 
 - Added sqlite-query dependency to xvc pipelines. A step is invalidated to run when the output from an SQLite query changes.
 - Moved bundled sqlite behind a feature flag, bundled_sqlite.
 - Merged xvc-workflow-tests crate to the main xvc crate.
-- Added compiling Xvc with non-default features to the book.
+- Added compiling Xvc with non-default features document
 
-### 0.6.8 (2024-07-15)
+## 0.6.8 (2024-07-15)
 
 - Modifications for Python bindings
 
@@ -76,7 +80,7 @@ tree, and subtasks are marked with indentation.
 
 - Major overhaul in pipelines
   - PR: <https://github.com/iesahin/xvc/pull/224/>
-  - Added `--generic`, `--url`, `--regex-items`, `--line-items`, `--glob-items`
+  - Added `--generic`, `--url`, `--regex-items`, `--line-items`, `--glob-items` dependencies
   - as dependency types to `xvc pipeline step dependency` command
   - Updated `xvc pipelines` to run the pipeline by creating a thread for each step.
   - Updated pipelines state machine

diff --git a/book/src/ref/xvc-pipeline-run.md b/book/src/ref/xvc-pipeline-run.md
@@ -56,7 +56,7 @@ You can run the default pipeline without specifying its name.
 $ xvc pipeline run
 [OUT] [hello] hello
 
-[DONE] hello (echo hello)
+[DONE] [hello] (echo hello)
 
 
 ```
@@ -85,7 +85,7 @@ $ xvc pipeline --pipeline-name my-pipeline step new --step-name my-hello --comma
 $ xvc pipeline run --pipeline-name my-pipeline
 [OUT] [my-hello] hello from my-pipeline
 
-[DONE] my-hello (echo 'hello from my-pipeline')
+[DONE] [my-hello] (echo 'hello from my-pipeline')
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-file.md b/book/src/ref/xvc-pipeline-step-dependency-file.md
@@ -27,7 +27,7 @@ When you run the command, it will print `data.txt has changed` if the file `data
 $ xvc pipeline run
 [OUT] [file-dependency] data.txt has changed
 
-[DONE] file-dependency (echo data.txt has changed)
+[DONE] [file-dependency] (echo data.txt has changed)
 
 
 ```
@@ -45,7 +45,7 @@ A step will run if any of its dependencies have changed.
 $ xvc pipeline run
 [OUT] [file-dependency] data.txt has changed
 
-[DONE] file-dependency (echo data.txt has changed)
+[DONE] [file-dependency] (echo data.txt has changed)
 
 
 ```
@@ -69,7 +69,7 @@ Now the step will run even if none of the dependencies have changed.
 $ xvc pipeline run
 [OUT] [file-dependency] data.txt has changed
 
-[DONE] file-dependency (echo data.txt has changed)
+[DONE] [file-dependency] (echo data.txt has changed)
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-generic.md b/book/src/ref/xvc-pipeline-step-dependency-generic.md
@@ -36,7 +36,7 @@ The step won't run until tomorrow, when `date +%F` changes.
 $ xvc pipeline run
 [OUT] [morning-message] Good Morning!
 
-[DONE] morning-message (echo 'Good Morning!')
+[DONE] [morning-message] (echo 'Good Morning!')
 
 
 ```
@@ -52,7 +52,7 @@ $ xvc pipeline step dependency --step-name directory-contents --generic 'ls'
 $ xvc pipeline run
 [OUT] [directory-contents] Files changed
 
-[DONE] directory-contents (echo 'Files changed')
+[DONE] [directory-contents] (echo 'Files changed')
 
 
 ```
@@ -66,7 +66,7 @@ $ xvc-test-helper generate-random-file new-file.txt
 $ xvc pipeline run
 [OUT] [directory-contents] Files changed
 
-[DONE] directory-contents (echo 'Files changed')
+[DONE] [directory-contents] (echo 'Files changed')
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-glob-items.md b/book/src/ref/xvc-pipeline-step-dependency-glob-items.md
@@ -63,7 +63,7 @@ dir-0002/file-0003.bin
 ### Changed Files:
 
 
-[DONE] files-changed (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}")
+[DONE] [files-changed] (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}")
 
 
 $ xvc pipeline run
@@ -83,7 +83,7 @@ dir-0001/file-0001.bin
 ### Changed Files:
 
 
-[DONE] files-changed (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}")
+[DONE] [files-changed] (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}")
 
 
 ```
@@ -101,7 +101,7 @@ $ xvc pipeline run
 ### Changed Files:
 dir-0001/file-0002.bin
 
-[DONE] files-changed (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}")
+[DONE] [files-changed] (echo "### Added Files:/n${XVC_ADDED_GLOB_ITEMS}/n### Removed Files:/n${XVC_REMOVED_GLOB_ITEMS}/n### Changed Files:/n${XVC_CHANGED_GLOB_ITEMS}")
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-glob.md b/book/src/ref/xvc-pipeline-step-dependency-glob.md
@@ -48,7 +48,7 @@ The step is invalidated when a file described by the glob is added, removed or c
 $ xvc pipeline run
 [OUT] [files-changed] Files have changed.
 
-[DONE] files-changed (echo 'Files have changed.')
+[DONE] [files-changed] (echo 'Files have changed.')
 
 
 $ xvc pipeline run
@@ -62,7 +62,7 @@ $ rm dir-0001/file-0001.bin
 $ xvc pipeline run
 [OUT] [files-changed] Files have changed.
 
-[DONE] files-changed (echo 'Files have changed.')
+[DONE] [files-changed] (echo 'Files have changed.')
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-line-items.md b/book/src/ref/xvc-pipeline-step-dependency-line-items.md
@@ -77,7 +77,7 @@ $ xvc pipeline run
 Removed Lines:
 
 
-[DONE] print-top-10 (echo "Added Lines:/n ${XVC_ADDED_LINE_ITEMS}/nRemoved Lines:/n${XVC_REMOVED_LINE_ITEMS}")
+[DONE] [print-top-10] (echo "Added Lines:/n ${XVC_ADDED_LINE_ITEMS}/nRemoved Lines:/n${XVC_REMOVED_LINE_ITEMS}")
 
 
 ```
@@ -105,7 +105,7 @@ $ xvc pipeline run
 Removed Lines:
 "Hank",       "M",   30,       71,      158
 
-[DONE] print-top-10 (echo "Added Lines:/n ${XVC_ADDED_LINE_ITEMS}/nRemoved Lines:/n${XVC_REMOVED_LINE_ITEMS}")
+[DONE] [print-top-10] (echo "Added Lines:/n ${XVC_ADDED_LINE_ITEMS}/nRemoved Lines:/n${XVC_REMOVED_LINE_ITEMS}")
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-lines.md b/book/src/ref/xvc-pipeline-step-dependency-lines.md
@@ -67,7 +67,7 @@ $ xvc pipeline run
 "Hank",       "M",   30,       71,      158
 "Ivan",       "M",   53,       72,      175
 
-[DONE] print-top-10 (head people.csv)
+[DONE] [print-top-10] (head people.csv)
 
 
 ``````
@@ -101,7 +101,7 @@ $ xvc pipeline run
 "Ferzan",       "M",   30,       71,      158
 "Ivan",       "M",   53,       72,      175
 
-[DONE] print-top-10 (head people.csv)
+[DONE] [print-top-10] (head people.csv)
 
 
 ```

diff --git a/book/src/ref/xvc-pipeline-step-dependency-param.md b/book/src/ref/xvc-pipeline-step-dependency-param.md
@@ -50,11 +50,11 @@ Run for the first time, as initially all dependencies are invalid:
 $ xvc pipeline run
 [OUT] [read-hyperparams] Update Hyperparameters
 
-[DONE] read-hyperparams (echo "Update Hyperparameters")
+[DONE] [read-hyperparams] (echo "Update Hyperparameters")
 
 [OUT] [read-database-config] Updated Database Configuration
 
-[DONE] read-database-config (echo "Updated Database Configuration")
+[DONE] [read-database-config] (echo "Updated Database Configuration")
 
 
 ```
@@ -77,7 +77,7 @@ $ perl -pi -e 's/5432/9876/g' myparams.yaml
 $ xvc pipeline run
 [OUT] [read-database-config] Updated Database Configuration
 
-[DONE] read-database-config (echo "Updated Database Configuration")
+[DONE] [read-database-config] (echo "Updated Database Configuration")
 
 
 ```

diff --git a/book/src/ref/xvc-pipeline-step-dependency-regex-items.md b/book/src/ref/xvc-pipeline-step-dependency-regex-items.md
@@ -77,7 +77,7 @@ $ xvc pipeline run
 "Omar",       "M",   38,       70,      145
 "Quin",       "M",   29,       71,      176
 
-[DONE] new-males (echo "New Males:/n ${XVC_ADDED_REGEX_ITEMS}")
+[DONE] [new-males] (echo "New Males:/n ${XVC_ADDED_REGEX_ITEMS}")
 
 [OUT] [new-females] New Females:
  "Elly",       "F",   30,       66,      124
@@ -88,7 +88,7 @@ $ xvc pipeline run
 "Page",       "F",   31,       67,      135
 "Ruth",       "F",   28,       65,      131
 
-[DONE] new-females (echo "New Females:/n ${XVC_ADDED_REGEX_ITEMS}")
+[DONE] [new-females] (echo "New Females:/n ${XVC_ADDED_REGEX_ITEMS}")
 
 
 ```
@@ -132,7 +132,7 @@ $ xvc pipeline run
 [OUT] [new-females] New Females:
  "Asude",      "F",   12,       55,      110
 
-[DONE] new-females (echo "New Females:/n ${XVC_ADDED_REGEX_ITEMS}")
+[DONE] [new-females] (echo "New Females:/n ${XVC_ADDED_REGEX_ITEMS}")
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-regex.md b/book/src/ref/xvc-pipeline-step-dependency-regex.md
@@ -57,7 +57,7 @@ When you run the pipeline initially, the steps are run.
 $ xvc pipeline run
 [OUT] [count-females] 7
 
-[DONE] count-females (grep -c '"F",' people.csv)
+[DONE] [count-females] (grep -c '"F",' people.csv)
 
 
 ``````
@@ -100,7 +100,7 @@ $ cat people.csv
 $ xvc pipeline run
 [OUT] [count-females] 8
 
-[DONE] count-females (grep -c '"F",' people.csv)
+[DONE] [count-females] (grep -c '"F",' people.csv)
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-sqlite-query.md b/book/src/ref/xvc-pipeline-step-dependency-sqlite-query.md
@@ -61,7 +61,7 @@ Let's run the step without a dependency first.
 $ xvc pipeline run
 [OUT] [average-age] 34.6666666666667
 
-[DONE] average-age (sqlite3 people.db 'SELECT AVG(Age) FROM People;')
+[DONE] [average-age] (sqlite3 people.db 'SELECT AVG(Age) FROM People;')
 
 
 ```
@@ -83,7 +83,7 @@ So, when the number of people in the table changes, the step will run. Initially
 $ xvc pipeline run
 [OUT] [average-age] 34.6666666666667
 
-[DONE] average-age (sqlite3 people.db 'SELECT AVG(Age) FROM People;')
+[DONE] [average-age] (sqlite3 people.db 'SELECT AVG(Age) FROM People;')
 
 
 ```
@@ -107,7 +107,7 @@ This time, the step will run again as the result from dependency query (`SELECT
 $ xvc pipeline run
 [OUT] [average-age] 33.3684210526316
 
-[DONE] average-age (sqlite3 people.db 'SELECT AVG(Age) FROM People;')
+[DONE] [average-age] (sqlite3 people.db 'SELECT AVG(Age) FROM People;')
 
 
 ```

diff --git a/book/src/ref/xvc-pipeline-step-dependency-step.md b/book/src/ref/xvc-pipeline-step-dependency-step.md
@@ -25,11 +25,11 @@ When run, the dependency will be run first and the step will be run after.
 $ xvc pipeline run
 [OUT] [hello] hello
 
-[DONE] hello (echo hello)
+[DONE] [hello] (echo hello)
 
 [OUT] [world] world
 
-[DONE] world (echo world)
+[DONE] [world] (echo world)
 
 
 ```
@@ -49,7 +49,7 @@ $ xvc pipeline step update --step-name world --when always
 $ xvc pipeline run
 [OUT] [world] world
 
-[DONE] world (echo world)
+[DONE] [world] (echo world)
 
 
 ```
diff --git a/book/src/ref/xvc-pipeline-step-dependency-url.md b/book/src/ref/xvc-pipeline-step-dependency-url.md
@@ -26,7 +26,7 @@ The step is invalidated when the page is updated.
 $ xvc pipeline run
 [OUT] [xvc-docs-update] Xvc docs updated!
 
-[DONE] xvc-docs-update (echo 'Xvc docs updated!')
+[DONE] [xvc-docs-update] (echo 'Xvc docs updated!')
 
 
 ```

diff --git a/book/src/ref/xvc-pipeline-step-dependency.md b/book/src/ref/xvc-pipeline-step-dependency.md
@@ -43,7 +43,9 @@ Options:
           The difference between this and the glob-items option is that the glob-items option keeps track of all matching files individually, but this option only keeps track of the matched files' digest. This dependency uses considerably less disk space.
 
       --param <PARAMS>
-          Add a parameter dependency to the step in the form filename.yaml::model.units . Can be used multiple times
+          Add a parameter dependency to the step in the form filename.yaml::model.units
+          
+          The file can be a JSON, TOML, or YAML file. You can specify hierarchical keys like my.dict.key
 
       --regex_items <REGEX_ITEMS>
           Add a regex dependency in the form filename.txt:/^regex/ . Can be used multiple times.