Merge #1633

1633: build: harden build by pre-fetching dependencies r=tiagolobocastro a=tiagolobocastro

Attempt to vendor the cargo deps for up to 25 times Supports linking the resulting derivation into a global location through env var CARGO_VENDOR_DIR, example:
> CARGO_VENDOR_DIR=/tmp ./scripts/release.sh --image ""
/nix/store/2x03mh7l1q0jx4xfsd3nw4h4ks0pxxya-cargo-vendor-dir
    Cargo vendored dependencies pre-fetched into /tmp/mayastor-io-engine/develop
after 1 attempt(s)

This can allow us to create a place holder in the jenkins nodes to keep the last dependencies for each main branch of each repo.

Co-authored-by: Tiago Castro <[email protected]>

nix/overlay.nix

@@ -14,5 +14,5 @@ self: super: rec {
   ms-buildenv = super.callPackage ./pkgs/ms-buildenv { };
   nvme-cli = super.callPackage ./pkgs/nvme-cli { };
   nvmet-cli = super.callPackage ./pkgs/nvmet-cli { };
-  units = (super.callPackage ./pkgs/io-engine/units.nix { });
+  units = (super.callPackage ./pkgs/io-engine/units.nix { inherit tag sourcer; });
 }

nix/pkgs/io-engine/cargo-package.nix

@@ -95,6 +95,9 @@ let
   };
 in
 {
+  cargoDeps = rustPlatform.importCargoLock {
+    lockFile = ../../../Cargo.lock;
+  };
   release = rustPlatform.buildRustPackage (buildProps // {
     cargoBuildFlags = "--bin io-engine --bin io-engine-client --bin casperf";
     buildType = "release";

nix/pkgs/io-engine/units.nix

@@ -1,36 +1,18 @@
 { stdenv
-, clang
-, dockerTools
-, e2fsprogs
 , lib
-, libaio
-, libspdk
-, libspdk-dev
-, udev
-, liburing
-, makeRustPlatform
-, numactl
-, openssl
-, pkg-config
-, protobuf
-, sources
-, xfsprogs
-, utillinux
-, llvmPackages
-, targetPackages
-, buildPackages
-, targetPlatform
 , pkgs
+, git
+, tag
 , sourcer
 }:
 let
   versionDrv = import ../../lib/version.nix { inherit lib stdenv git tag sourcer; };
   versions = {
-    "version" = version;
+    "version" = "${versionDrv}";
     "long" = builtins.readFile "${versionDrv.long}";
     "tag_or_long" = builtins.readFile "${versionDrv.tag_or_long}";
   };
-  project-builder = { cargoBuildFlags }: pkgs.callPackage ./cargo-package.nix { inherit versions cargoBuildFlags; };
+  project-builder = { cargoBuildFlags ? [ ] }: pkgs.callPackage ./cargo-package.nix { inherit versions cargoBuildFlags; };
   components = { build }: {
     io-engine = (project-builder { cargoBuildFlags = [ "--bin io-engine" ]; }).${build};
     io-engine-cli = (project-builder { cargoBuildFlags = [ "--bin io-engine-cli" ]; }).${build};
@@ -39,6 +21,7 @@ let
   };
 in
 {
+  cargoDeps = (project-builder { }).cargoDeps;
   release = components { build = "release"; };
   debug = components { build = "debug"; };
 }

scripts/release.sh

@@ -30,6 +30,39 @@ nix_experimental() {
       echo -n " "
   fi
 }
+pre_fetch_cargo_deps() {
+  local nixAttrPath=$1
+  local project=$2
+  local maxAttempt=$3
+
+  local outLink="--no-out-link"
+  local cargoVendorMsg=""
+  if [ -n "$CARGO_VENDOR_DIR" ]; then
+    if [ "$(realpath -s "$CARGO_VENDOR_DIR")" = "$(realpath -s "$SCRIPTDIR/..")" ]; then
+      cargoVendorDir="$CARGO_VENDOR_DIR/$GIT_BRANCH"
+    else
+      cargoVendorDir="$CARGO_VENDOR_DIR/$project/$GIT_BRANCH"
+    fi
+
+    outLink="--out-link "$cargoVendorDir""
+    cargoVendorMsg="into $(realpath -s "$cargoVendorDir") "
+  fi
+
+  for (( attempt=1; attempt<=maxAttempt; attempt++ )); do
+     if $NIX_BUILD $outLink -A "$nixAttrPath"; then
+       echo "Cargo vendored dependencies pre-fetched "$cargoVendorMsg"after $attempt attempt(s)"
+       return 0
+     fi
+     sleep 1
+  done
+  if [ "$attempt" = "1" ]; then
+    echo "Cargo vendor pre-fetch is disabled"
+    return 0
+  fi
+
+  echo "Failed to pre-fetch the cargo vendored dependencies in $maxAttempt attempts"
+  exit 1
+}
 
 help() {
   cat <<EOF
@@ -59,8 +92,8 @@ RM="rm"
 SCRIPTDIR=$(dirname "$0")
 TAG=`get_tag`
 HASH=`get_hash`
-BRANCH=`git rev-parse --abbrev-ref HEAD`
-BRANCH=${BRANCH////-}
+GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+BRANCH=${GIT_BRANCH////-}
 IMAGES=
 UPLOAD=
 SKIP_PUBLISH=
@@ -69,6 +102,8 @@ REGISTRY=
 ALIAS=
 DEBUG=
 OVERRIDE_COMMIT_HASH=
+CARGO_VENDOR_DIR=${CARGO_VENDOR_DIR:-}
+CARGO_VENDOR_ATTEMPTS=${CARGO_VENDOR_ATTEMPTS:-25}
 
 # Check if all needed tools are installed
 curl --version >/dev/null
@@ -140,6 +175,9 @@ done
 
 cd $SCRIPTDIR/..
 
+# pre-fetch build dependencies with a number of attempts to harden against flaky networks
+pre_fetch_cargo_deps units.cargoDeps "mayastor-io-engine" "$CARGO_VENDOR_ATTEMPTS"
+
 if [ -z "$IMAGES" ]; then
   if [ -z "$DEBUG" ]; then
     IMAGES="mayastor-io-engine mayastor-fio-spdk mayastor-casperf"