Removed unnecessary spawn

allenai · Oct 29, 2023 · 7c9f08e · 7c9f08e
1 parent 2ee1ae2
commit 7c9f08e
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/python/dolma/tokenizer/executor.py b/python/dolma/tokenizer/executor.py
@@ -188,8 +188,6 @@ def tokenize_in_parallel(
     dtype: str = "uint16",
     debug: bool = False,
 ):
-    multiprocessing.set_start_method("spawn")
-
     # variables for the nice debugging and tokenizers
     os.environ["PYTHONBREAKPOINT"] = "ipdb.set_trace"
     os.environ["TOKENIZERS_PARALLELISM"] = "false"