Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

twoliter: allow partial lockfile validation in some scenarios #361

Merged
merged 4 commits into from
Sep 6, 2024
+743 −212
Merged

twoliter: allow partial lockfile validation in some scenarios #361

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c451e4a
twoliter: only run build-kit tests during integ
cbgbt Sep 6, 2024
87624fe
twoliter: allow partial lockfile validation in some scenarios
cbgbt Sep 6, 2024
8cf1e40
twoliter: improve callsite clarity of ImageResolver
cbgbt Sep 6, 2024
4703ab6
Makefile.toml: do not fetch-sources unnecessarily
cbgbt Sep 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ sha2 = "0.10"
shell-words = "1"
simplelog = "0.12"
snafu = "0.8"
strum = "0.26"
tabled = "0.10"
tar = "0.4"
tempfile = "3"
Expand Down
1 change: 1 addition & 0 deletions twoliter/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ semver = { workspace = true, features = ["serde"] }
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
sha2.workspace = true
strum = { workspace = true, features = ["derive"] }
tar.workspace = true
tempfile.workspace = true
tokio = { workspace = true, features = ["fs", "macros", "process", "rt-multi-thread"] }
Expand Down
76 changes: 26 additions & 50 deletions twoliter/embedded/Makefile.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,10 @@ dependencies = ["setup-build"]
script_runner = "bash"
script = [
'''
if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.sdk-verified" ]; then
echo "Twoliter could not validate '${TLPRIVATE_SDK_IMAGE}', refusing to continue" >&2
exit 1
fi
if ! docker image inspect "${TLPRIVATE_SDK_IMAGE}" >/dev/null 2>&1 ; then
if ! docker pull "${TLPRIVATE_SDK_IMAGE}" ; then
echo "failed to pull '${TLPRIVATE_SDK_IMAGE}'" >&2
Expand Down Expand Up @@ -736,6 +740,18 @@ docker run --rm \
'''
]

[tasks.validate-kits]
dependencies = ["cargo-metadata"]
script_runner = "bash"
script = [
'''
if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.kits-verified" ]; then
echo "Twoliter could not validate external kits, refusing to continue" >&2
exit 1
fi
'''
]

# Reads the project's workspace Cargo dependency graph to a json file. Needed by buildsys when
# building packages, kits and variants.
[tasks.cargo-metadata]
Expand Down Expand Up @@ -772,7 +788,7 @@ cargo metadata \

# Builds a package including its build-time and runtime dependency packages.
[tasks.build-package]
dependencies = ["check-cargo-version", "fetch", "publish-setup", "cargo-metadata"]
dependencies = ["check-cargo-version", "fetch", "publish-setup", "validate-kits"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -817,7 +833,7 @@ cargo build \

# Builds a kit including its dependency packages.
[tasks.build-kit]
dependencies = ["check-cargo-version", "fetch", "publish-setup", "cargo-metadata"]
dependencies = ["check-cargo-version", "fetch", "publish-setup", "validate-kits"]
script_runner = "bash"
script = [
'''
Expand All @@ -841,7 +857,7 @@ cargo build \
]

[tasks.build-variant]
dependencies = ["fetch", "build-sbkeys", "publish-setup", "cargo-metadata"]
dependencies = ["fetch", "build-sbkeys", "publish-setup", "validate-kits"]
script = [
'''
export PATH="${TWOLITER_TOOLS_DIR}:${PATH}"
Expand Down Expand Up @@ -892,7 +908,7 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${OUTPUT_LOGS_DIR}/latest"
]

[tasks.build-all]
dependencies = ["fetch", "build-sbkeys", "publish-setup", "cargo-metadata"]
dependencies = ["fetch", "build-sbkeys", "publish-setup", "validate-kits"]
script = [
'''
export PATH="${TWOLITER_TOOLS_DIR}:${PATH}"
Expand Down Expand Up @@ -968,10 +984,7 @@ run_task = "publish-setup"
# to create a repo under /build/repos, named after the arch/variant/version,
# containing subdirectories for the repo metadata and targets.
[tasks.repo]
# Rather than depend on "build", which currently rebuilds images each run, we
# check for the image files below to save time. This does mean that `cargo
# make` must be run before `cargo make repo`.
dependencies = ["publish-setup", "fetch-sources"]
dependencies = ["publish-setup"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1068,7 +1081,7 @@ ln -sfn "${PUBLISH_REPO_OUTPUT_DIR##*/}" "${PUBLISH_REPO_OUTPUT_DIR%/*}/latest"
]

[tasks.validate-repo]
dependencies = ["publish-setup-without-key", "fetch-sources"]
dependencies = ["publish-setup-without-key"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1134,7 +1147,7 @@ run_task = "fetch-variant"
run_task = "fetch-friendly-variant"

[tasks.check-repo-expirations]
dependencies = ["publish-setup-without-key", "fetch-sources"]
dependencies = ["publish-setup-without-key"]
script_runner = "bash"
script = [
'''
Expand All @@ -1158,7 +1171,7 @@ pubsys \
]

[tasks.refresh-repo]
dependencies = ["publish-setup", "fetch-sources"]
dependencies = ["publish-setup"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1189,10 +1202,7 @@ pubsys \
]

[tasks.ami]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the image files below to save time.
# This does mean that `cargo make` must be run before `cargo make ami`.
dependencies = ["setup-build", "fetch-sources"]
dependencies = ["setup-build"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1262,10 +1272,6 @@ ln -snf "${ami_output##*/}" "${ami_output_latest}"
]

[tasks.ami-public]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the input file below to save time.
# This does mean that `cargo make ami` must be run before `cargo make ami-public`.
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1293,10 +1299,6 @@ pubsys \
]

[tasks.ami-private]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the input file below to save time.
# This does mean that `cargo make ami` must be run before `cargo make ami-private`.
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1324,10 +1326,6 @@ pubsys \
]

[tasks.grant-ami]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the input file below to save time.
# This does mean that `cargo make ami` must be run before `cargo make grant-ami`.
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1363,10 +1361,6 @@ pubsys \
]

[tasks.revoke-ami]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the input file below to save time.
# This does mean that `cargo make ami` must be run before `cargo make revoke-ami`.
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1402,10 +1396,6 @@ pubsys \
]

[tasks.validate-ami]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the input file below to save time.
# This does mean that `cargo make ami` must be run before `cargo make validate-ami`.
dependencies = ["fetch-sources"]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like we can get rid of most of these 'Rather than depend on "build"' comments now, since these tasks don't depend on "publish-tools" or anything else.

script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1433,10 +1423,6 @@ pubsys \
]

[tasks.ssm]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the input file below to save time.
# This does mean that `cargo make ami` must be run before `cargo make ssm`.
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1471,7 +1457,6 @@ pubsys \
]

[tasks.promote-ssm]
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1506,10 +1491,6 @@ pubsys \
]

[tasks.validate-ssm]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the input file below to save time.
# This does mean that `cargo make ssm` must be run before `cargo make validate-ssm`.
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1537,7 +1518,6 @@ pubsys \
]

[tasks.publish-kit]
dependencies = ["fetch-sources"]
script_runner = "bash"
script = [
'''
Expand Down Expand Up @@ -1567,11 +1547,7 @@ pubsys \
]

[tasks._upload-ova-base]
# Rather than depend on "build", which currently rebuilds images each run, we
# depend on publish-tools and check for the image files below to save time.
# This does mean that `cargo make` must be run before
# `cargo make _upload-ova-base`.
dependencies = ["setup-build", "fetch-sources"]
dependencies = ["setup-build"]
script_runner = "bash"
script = [
'''
Expand Down
Loading
Loading