From c3ad10cd7abda63e49ab7b337e964fc63771a481 Mon Sep 17 00:00:00 2001 From: Qiying Wang <781345688@qq.com> Date: Mon, 6 Mar 2023 21:52:04 +0800 Subject: [PATCH 1/2] Update llama.sh Use aria2c to for parallel downloading and continuation --- llama.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.sh b/llama.sh index 6308ce0..99a2a81 100755 --- a/llama.sh +++ b/llama.sh @@ -30,7 +30,7 @@ do mkdir -p ${TARGET_FOLDER}"/${i}" for s in $(seq -f "0%g" 0 ${N_SHARD_DICT[$i]}) do - wget ${PRESIGNED_URL/'*'/"${i}/consolidated.${s}.pth"} -O ${TARGET_FOLDER}"/${i}/consolidated.${s}.pth" + aria2c -c -s10 -x10 -k1M ${PRESIGNED_URL/'*'/"${i}/consolidated.${s}.pth"} -o ${TARGET_FOLDER}"/${i}/consolidated.${s}.pth" done wget ${PRESIGNED_URL/'*'/"${i}/params.json"} -O ${TARGET_FOLDER}"/${i}/params.json" wget ${PRESIGNED_URL/'*'/"${i}/checklist.chk"} -O ${TARGET_FOLDER}"/${i}/checklist.chk" From 8a8f824b4c1df4258bbe13a71fc8dba0b60b6095 Mon Sep 17 00:00:00 2001 From: WqyJh <781345688@qq.com> Date: Thu, 9 Mar 2023 11:28:43 +0800 Subject: [PATCH 2/2] Use wget as fallback --- llama.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llama.sh b/llama.sh index 99a2a81..938988e 100755 --- a/llama.sh +++ b/llama.sh @@ -30,7 +30,11 @@ do mkdir -p ${TARGET_FOLDER}"/${i}" for s in $(seq -f "0%g" 0 ${N_SHARD_DICT[$i]}) do - aria2c -c -s10 -x10 -k1M ${PRESIGNED_URL/'*'/"${i}/consolidated.${s}.pth"} -o ${TARGET_FOLDER}"/${i}/consolidated.${s}.pth" + if ! which aria2c 2>&1 > /dev/null; then + wget ${PRESIGNED_URL/'*'/"${i}/consolidated.${s}.pth"} -O ${TARGET_FOLDER}"/${i}/consolidated.${s}.pth" + else + aria2c -c -s10 -x10 -k1M ${PRESIGNED_URL/'*'/"${i}/consolidated.${s}.pth"} -o ${TARGET_FOLDER}"/${i}/consolidated.${s}.pth" + fi done wget ${PRESIGNED_URL/'*'/"${i}/params.json"} -O ${TARGET_FOLDER}"/${i}/params.json" wget ${PRESIGNED_URL/'*'/"${i}/checklist.chk"} -O ${TARGET_FOLDER}"/${i}/checklist.chk"