feat: introduce HFPath

megvii-research · Nov 1, 2024 · 1f75833 · 1f75833
1 parent 300eb9a
commit 1f75833
Show file tree

Hide file tree

Showing 5 changed files with 614 additions and 1 deletion.
diff --git a/megfile/__init__.py b/megfile/__init__.py
@@ -65,6 +65,7 @@
     is_hdfs,
 )
 from megfile.hdfs_path import HdfsPath
+from megfile.hf_path import HFPath
 from megfile.http import (
     http_exists,
     http_getmtime,
@@ -396,6 +397,7 @@
     "hdfs_makedirs",
     "S3Path",
     "FSPath",
+    "HFPath",
     "HttpPath",
     "HttpsPath",
     "StdioPath",

diff --git a/megfile/config.py b/megfile/config.py
@@ -60,3 +60,6 @@
 SFTP_MAX_RETRY_TIMES = int(
     os.getenv("MEGFILE_SFTP_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
 )
+HF_MAX_RETRY_TIMES = int(
+    os.getenv("MEGFILE_HF_MAX_RETRY_TIMES") or DEFAULT_MAX_RETRY_TIMES
+)