From 7e438b3fd651664247a1c90ce95bc40f0371c1f1 Mon Sep 17 00:00:00 2001 From: Amir Pourmand Date: Wed, 2 Aug 2023 09:41:49 +0000 Subject: [PATCH] add wikipedia-fa --- data/datasets/__init__.py | 1 + data/datasets/fa-wikipedia/README.md | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 data/datasets/fa-wikipedia/README.md diff --git a/data/datasets/__init__.py b/data/datasets/__init__.py index 6cf9e7f027..92a82179f0 100644 --- a/data/datasets/__init__.py +++ b/data/datasets/__init__.py @@ -4,6 +4,7 @@ "tv_dialogue": "sedthh/tv_dialogue", # TV and Movie dialogues and transcripts "fd_dialogue": "sedthh/fd_dialogue", # TV and Movie dialogues and transcripts from ForeverDreaming "tlcv2.0_oa": "pythainlp/tlcv2.0_oa", # Thai classical literature texts + "fa-wikipedia": "pourmand1376/fa-wikipedia", # Farsi Wikipedia texts } INSTRUCTION_DATASETS = { diff --git a/data/datasets/fa-wikipedia/README.md b/data/datasets/fa-wikipedia/README.md new file mode 100644 index 0000000000..728190a737 --- /dev/null +++ b/data/datasets/fa-wikipedia/README.md @@ -0,0 +1,6 @@ +This dataset is crawled from +[farsi wikipedia](https://fa.wikipedia.org/wiki/%D8%B5%D9%81%D8%AD%D9%87%D9%94_%D8%A7%D8%B5%D9%84%DB%8C). +This is valuable clean text data in persian (Farsi). It contains information +about all subjects. + +It has 2.53M Articles.