Skip to content

Commit

Permalink
Use tdnf cache mount for package cache
Browse files Browse the repository at this point in the history 
Before we were using (but not always) the perisistent tndf cache at
`/var/cache/tdnf` and then also caching rpms a 2nd time at
`/root/.cache/...`.
This effectively gave us 2 caches of the same packages which is
uneccessary.
It also means we had to use flock to prevent multiple writers to the
2nd cache (something the tdnf cache would already be doing).

This changes things so we just use one cache, the tdnf cache, and the
rpms we feed into the toolkit chroot only contains the rpms needed for
that build.

Due to this change I noticed the toolkit actually wants both build and
runtime dependencies because builds started failing due to missing
dependencies (since we were no longer using a huge pool of rpms).
So along with this change the package download now includes both build
and runtime deps.

Finally, since I was already messing with the install scripts I moved
the multi-line commands we were feeding into buildkit into mounted files
that we execute.
The reason for this is purely cosmetic.
The buildkit terminal output does not like our multi-line command
strings.

Signed-off-by: Brian Goff <[email protected]>
  • Loading branch information
@cpuguy83
cpuguy83 committed Oct 31, 2023
1 parent ae38f18 commit 1837b90
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 49 deletions.
7 changes: 5 additions & 2 deletions frontend/mariner2/target_container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func specToContainerLLB(spec *dalec.Spec, target string, builderImg llb.State, r
chrootedPathEnv := strings.Join(chrootedPaths, ":")

installCmd := `
set -x
#!/usr/bin/env sh
check_non_empty() {
ls ${1} > /dev/null 2>&1
Expand Down Expand Up @@ -202,12 +202,15 @@ rpm --dbpath=` + rpmdbDir + ` -qa --qf "%{NAME}\t%{VERSION}-%{RELEASE}\t%{INSTAL
rm -rf ` + rpmdbDir + `
`

installer := llb.Scratch().File(llb.Mkfile("install.sh", 0o755, []byte(installCmd)))

baseImg := llb.Image(getBaseOutputImage(spec, target), llb.WithMetaResolver(sOpt.Resolver))
worker := builderImg.
Run(
shArgs(installCmd),
shArgs("/tmp/install.sh"),
marinerTdnfCache,
llb.AddMount("/tmp/rpms", rpmDir, llb.SourcePath("/RPMS")),
llb.AddMount("/tmp/install.sh", installer, llb.SourcePath("install.sh")),
)

// This adds a mount to the worker so that all the commands are run with this mount added
Expand Down
81 changes: 34 additions & 47 deletions frontend/mariner2/target_rpm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,6 @@ const (

var (
marinerTdnfCache = llb.AddMount("/var/cache/tdnf", llb.Scratch(), llb.AsPersistentCacheDir("mariner2-tdnf-cache", llb.CacheMountLocked))
// Setup a lockfile so that any instances sharing the rpms cache mount can take a lock before downloading rpms.
// We don't want multiple things writing to the cache at the same time, nor do we want a reader to read a file that is being written to.
rpmLockFile = llb.AddMount("/rpmcachelock", llb.Scratch(), llb.AsPersistentCacheDir("dalec-mariner2-rpmcachelock", llb.CacheMountShared))
rpmLockFilePath = "/rpmcachelock/lock"

cachedRPMsMount = llb.AddMount(filepath.Join(cachedRpmsDir, "cache"), llb.Scratch(), llb.AsPersistentCacheDir(cachedRpmsName, llb.CacheMountShared))
)

func handleRPM(ctx context.Context, client gwclient.Client, spec *dalec.Spec) (gwclient.Reference, *image.Image, error) {
Expand Down Expand Up @@ -138,20 +132,19 @@ func getBaseBuilderImg(ctx context.Context, client gwclient.Client) (llb.State,
return llb.Image(toolchainImgRef, llb.WithMetaResolver(client)), nil
}

// withRpmsLock wraps the command in an flock call with either a shared or exclusive lock.
// See https://linux.die.net/man/1/flock for more details and examples.
func withRpmsLock(cmd string, exclusive bool) string {
lockFl := "-s" // shared lock
if exclusive {
lockFl = "-x" // exclusive lock
}
return fmt.Sprintf(`
const cleanupScript = `
#!/usr/bin/env sh
for i in "${CHROOT_DIR}/"*; do
(
set -e
flock %s 42
%s
) 42>%s
`, lockFl, cmd, rpmLockFilePath)
if [ -d "${i}" ]; then
cd "${i}"; find . ! -path "./upstream-cached-rpms/*" ! -path "./upstream-cached-rpms" ! -path "." -delete -print || exit 42
fi
)
done
`

func getAllDeps(spec *dalec.Spec) []string {
return sort.StringSlice(append(getBuildDeps(spec), getRuntimeDeps(spec)...))
}

func specToRpmLLB(spec *dalec.Spec, getDigest getDigestFunc, baseImg llb.State, sOpt dalec.SourceOpts) (llb.State, error) {
Expand All @@ -175,6 +168,22 @@ func specToRpmLLB(spec *dalec.Spec, getDigest getDigestFunc, baseImg llb.State,
AddEnv("LOG_LEVEL", "debug").
AddEnv("CACHED_RPMS_DIR", cachedRpmsDir)

specsMount := llb.AddMount(specsDir, br, llb.SourcePath("/SPECS"))

cachedRpms := llb.Scratch()
cacheDir := filepath.Join(cachedRpmsDir, "cache")

// The mariner toolkit is trying to resolve *all* dependencies and not just build dependencies.
// We need to make sure we have all the dependencies cached otherwise the build will fail.
if deps := getAllDeps(spec); len(deps) > 0 {
dlCmd := "tdnf install --downloadonly -y --releasever=2.0 --alldeps --downloaddir \"${CACHED_RPMS_DIR}/cache\" " + strings.Join(deps, " ")
cachedRpms = work.
Run(
shArgs(dlCmd),
marinerTdnfCache,
).AddMount(cacheDir, llb.Scratch())
}

prepareChroot := runOptFunc(func(ei *llb.ExecInfo) {
// Mount cached packages into the chroot dirs so they are available to the chrooted build.
// The toolchain has built-in (yum) repo files that points to "/upstream-cached-rpms",
Expand All @@ -189,45 +198,23 @@ func specToRpmLLB(spec *dalec.Spec, getDigest getDigestFunc, baseImg llb.State,
return filepath.Join("/tmp/chroot", "dalec"+strconv.Itoa(i), p)
}
for i := 0; i < runtime.NumCPU(); i++ {
llb.AddMount(dir(i, "upstream-cached-rpms"), llb.Scratch(), llb.AsPersistentCacheDir(cachedRpmsName, llb.CacheMountShared)).SetRunOption(ei)
llb.AddMount(dir(i, "upstream-cached-rpms"), cachedRpms).SetRunOption(ei)
}
})

specsMount := llb.AddMount(specsDir, br, llb.SourcePath("/SPECS"))

dlCmd := withRpmsLock("tdnf install --downloadonly -y --releasever=2.0 --alldeps --downloaddir \"${CACHED_RPMS_DIR}/cache\" "+strings.Join(getBuildDeps(spec), " "), true)
buildCmd := withRpmsLock(`
make -j$(nproc) build-packages || (cat /build/build/logs/pkggen/rpmbuilding/*; exit 1)
for i in "${CHROOT_DIR}/"*; do
(
if [ -d "${i}" ]; then
cd "${i}"; find . ! -path "./upstream-cached-rpms/*" ! -path "./upstream-cached-rpms" ! -path "." -delete -print || exit 42
fi
)
done
`, false)

worker := work.With(func(st llb.State) llb.State {
deps := getBuildDeps(spec)
if len(deps) == 0 {
return st
}
return st.Run(
shArgs(dlCmd),
cachedRPMsMount,
rpmLockFile,
).State
}).
cleanupScript := llb.Scratch().File(llb.Mkfile("chroot-cleanup.sh", 0o755, []byte(cleanupScript)))
buildCmd := `trap '/tmp/chroot-cleanup.sh > /dev/null' EXIT; make -j$(nproc) build-packages || (cat /build/build/logs/pkggen/rpmbuilding/*; ls -lh ${CACHED_RPMS_DIR}/cache; exit 1)`
worker := work.
Run(
shArgs(buildCmd),
prepareChroot,
cachedRPMsMount,
specsMount,
rpmLockFile,
marinerTdnfCache,
llb.AddEnv("VERSION", spec.Version),
llb.AddEnv("BUILD_NUMBER", spec.Revision),
llb.AddEnv("REFRESH_WORKER_CHROOT", "n"),
llb.AddMount(cacheDir, cachedRpms),
llb.AddMount("/tmp/chroot-cleanup.sh", cleanupScript, llb.SourcePath("chroot-cleanup.sh")),
)

st := worker.
Expand Down

0 comments on commit 1837b90

Please sign in to comment.