add wikipedia-fa

LAION-AI · Aug 2, 2023 · 7e438b3 · 7e438b3
1 parent aa96100
commit 7e438b3
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/data/datasets/__init__.py b/data/datasets/__init__.py
@@ -4,6 +4,7 @@
     "tv_dialogue": "sedthh/tv_dialogue",  # TV and Movie dialogues and transcripts
     "fd_dialogue": "sedthh/fd_dialogue",  # TV and Movie dialogues and transcripts from ForeverDreaming
     "tlcv2.0_oa": "pythainlp/tlcv2.0_oa",  # Thai classical literature texts
+    "fa-wikipedia": "pourmand1376/fa-wikipedia",  # Farsi Wikipedia texts
 }
 
 INSTRUCTION_DATASETS = {

diff --git a/data/datasets/fa-wikipedia/README.md b/data/datasets/fa-wikipedia/README.md
@@ -0,0 +1,6 @@
+This dataset is crawled from
+[farsi wikipedia](https://fa.wikipedia.org/wiki/%D8%B5%D9%81%D8%AD%D9%87%D9%94_%D8%A7%D8%B5%D9%84%DB%8C).
+This is valuable clean text data in persian (Farsi). It contains information
+about all subjects.
+
+It has 2.53M Articles.