minor fix to HF mirror script (#1962)

* minor fix to HF mirror script * fix order * fix_name * update uploaded preset json
keras-team · Oct 30, 2024 · 7e39d97 · 7e39d97
1 parent 97f344a
commit 7e39d97
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 17 deletions.
diff --git a/tools/hf_uploaded_presets.json b/tools/hf_uploaded_presets.json
@@ -3,6 +3,8 @@
     "kaggle://keras/densenet/keras/densenet_121_imagenet/2",
     "kaggle://keras/densenet/keras/densenet_169_imagenet/2",
     "kaggle://keras/densenet/keras/densenet_201_imagenet/2",
+    "kaggle://keras/efficientnet/keras/efficientnet_b0_ra_imagenet",
+    "kaggle://keras/efficientnet/keras/efficientnet_b1_ft_imagenet",
     "kaggle://keras/mit/keras/mit_b0_ade20k_512/1",
     "kaggle://keras/mit/keras/mit_b1_ade20k_512/1",
     "kaggle://keras/mit/keras/mit_b2_ade20k_512/1",
@@ -45,21 +47,31 @@
     "kaggle://keras/resnetv1/keras/resnet_152_imagenet/2",
     "kaggle://keras/resnetv2/keras/resnet_v2_50_imagenet/2",
     "kaggle://keras/resnetv2/keras/resnet_v2_101_imagenet/2",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_18_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_34_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_50_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_50_ssld_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_50_ssld_v2_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_50_ssld_v2_fix_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_101_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_101_ssld_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_152_imagenet",
+    "kaggle://keras/resnet_vd/keras/resnet_vd_200_imagenet",
     "kaggle://keras/sam/keras/sam_base_sa1b/4",
     "kaggle://keras/sam/keras/sam_large_sa1b/4",
     "kaggle://keras/sam/keras/sam_huge_sa1b/4",
-    "kaggle://kerashub/segformer/keras/segformer_b0_ade20k_512",
-    "kaggle://kerashub/segformer/keras/segformer_b1_ade20k_512",
-    "kaggle://kerashub/segformer/keras/segformer_b2_ade20k_512",
-    "kaggle://kerashub/segformer/keras/segformer_b3_ade20k_512",
-    "kaggle://kerashub/segformer/keras/segformer_b4_ade20k_512",
-    "kaggle://kerashub/segformer/keras/segformer_b5_ade20k_640",
-    "kaggle://kerashub/segformer/keras/segformer_b0_cityscapes_1024",
-    "kaggle://kerashub/segformer/keras/segformer_b1_ade20k_512",
-    "kaggle://kerashub/segformer/keras/segformer_b2_cityscapes_1024",
-    "kaggle://kerashub/segformer/keras/segformer_b3_cityscapes_1024",
-    "kaggle://kerashub/segformer/keras/segformer_b4_cityscapes_1024",
-    "kaggle://kerashub/segformer/keras/segformer_b5_cityscapes_1024",
+    "kaggle://keras/segformer/keras/segformer_b0_ade20k_512",
+    "kaggle://keras/segformer/keras/segformer_b1_ade20k_512",
+    "kaggle://keras/segformer/keras/segformer_b2_ade20k_512",
+    "kaggle://keras/segformer/keras/segformer_b3_ade20k_512",
+    "kaggle://keras/segformer/keras/segformer_b4_ade20k_512",
+    "kaggle://keras/segformer/keras/segformer_b5_ade20k_640",
+    "kaggle://keras/segformer/keras/segformer_b0_cityscapes_1024",
+    "kaggle://keras/segformer/keras/segformer_b1_ade20k_512",
+    "kaggle://keras/segformer/keras/segformer_b2_cityscapes_1024",
+    "kaggle://keras/segformer/keras/segformer_b3_cityscapes_1024",
+    "kaggle://keras/segformer/keras/segformer_b4_cityscapes_1024",
+    "kaggle://keras/segformer/keras/segformer_b5_cityscapes_1024",
     "kaggle://keras/vgg/keras/vgg_11_imagenet/1",
     "kaggle://keras/vgg/keras/vgg_13_imagenet/1",
     "kaggle://keras/vgg/keras/vgg_16_imagenet/1",
@@ -138,12 +150,19 @@
     "kaggle://keras/roberta/keras/roberta_base_en/2",
     "kaggle://keras/roberta/keras/roberta_large_en/2",
     "kaggle://keras/stablediffusion3/keras/stable_diffusion_3_medium/3",
+    "kaggle://keras/stablediffusion-3.5/keras/stable_diffusion_3.5_large/1",
+    "kaggle://keras/stablediffusion-3.5/keras/stable_diffusion_3.5_large_turbo/1",
     "kaggle://keras/t5/keras/t5_small_multi/2",
+    "kaggle://keras/t5/keras/t5_1.1_small/1",
     "kaggle://keras/t5/keras/t5_base_multi/2",
+    "kaggle://keras/t5/keras/t5_1.1_base/1",
     "kaggle://keras/t5/keras/t5_large_multi/2",
+    "kaggle://keras/t5/keras/t5_1.1_large/1",
+    "kaggle://keras/t5/keras/t5_1.1_xl/1",
+    "kaggle://keras/t5/keras/t5_1.1_xxl/1",
     "kaggle://keras/t5/keras/flan_small_multi/2",
     "kaggle://keras/t5/keras/flan_base_multi/2",
     "kaggle://keras/t5/keras/flan_large_multi/2",
     "kaggle://keras/xlm_roberta/keras/xlm_roberta_base_multi/2",
-    "kaggle://keras/xlm_roberta/keras/xlm_roberta_large_multi/2",
+    "kaggle://keras/xlm_roberta/keras/xlm_roberta_large_multi/2"
 ]
diff --git a/tools/mirror_weights_on_hf.py b/tools/mirror_weights_on_hf.py
@@ -26,10 +26,9 @@ def download_and_upload_missing_models(missing_in_hf_uploads):
     errored_uploads = []
     for kaggle_handle in missing_in_hf_uploads:
         try:
-            model_variant = kaggle_handle.split("/")[3]
-            hf_uri = f"{HF_BASE_URI}/{model_variant}"
             kaggle_handle_path = kaggle_handle.removeprefix("kaggle://")
-
+            model_variant = kaggle_handle_path.split("/")[3]
+            hf_uri = f"{HF_BASE_URI}/{model_variant}"
             # Skip Gemma models
             if "gemma" in kaggle_handle_path:
                 print(f"Skipping Gemma model preset: {kaggle_handle_path}")
@@ -87,7 +86,7 @@ def main():
 
     # Step 5: Update JSON file with newly uploaded handles
     update_hf_uploads_json(
-        JSON_FILE_PATH, {latest_kaggle_handles} - {errored_uploads}
+        JSON_FILE_PATH, list(set(latest_kaggle_handles) - set(errored_uploads))
     )
     print("uploads for the following models failed: ", errored_uploads)
     print("Rest of the models up to date on HuggingFace")