diff --git a/colossalai/inference/core/llm_engine.py b/colossalai/inference/core/llm_engine.py
index b973d371dac7..1dbc3ace85b6 100644
--- a/colossalai/inference/core/llm_engine.py
+++ b/colossalai/inference/core/llm_engine.py
@@ -57,11 +57,11 @@ class LLMEngine(BaseEngine):
 
     def __init__(
         self,
-        model_or_path: nn.Module | str,
-        tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast = None,
+        model_or_path: Union[nn.Module, str],
+        tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast] = None,
         inference_config: InferenceConfig = None,
         verbose: bool = False,
-        model_policy: Policy | type[Policy] = None,
+        model_policy: Union[Policy, type[Policy]] = None,
     ) -> None:
         self.inference_config = inference_config
         self.dtype = inference_config.dtype
diff --git a/colossalai/inference/utils.py b/colossalai/inference/utils.py
index f2a0fc0370c1..d0851e362318 100644
--- a/colossalai/inference/utils.py
+++ b/colossalai/inference/utils.py
@@ -186,8 +186,6 @@ def get_model_type(model_or_path: Union[nn.Module, str, DiffusionPipeline]):
             """
 
         try:
-            from diffusers import DiffusionPipeline
-
             DiffusionPipeline.load_config(model_or_path)
             return ModelType.DIFFUSION_MODEL
         except:
diff --git a/examples/inference/stable_diffusion/test_ci.sh b/examples/inference/stable_diffusion/test_ci.sh
new file mode 100644
index 000000000000..d0189431cb20
--- /dev/null
+++ b/examples/inference/stable_diffusion/test_ci.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "Skip the test (this test is slow)"
diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt
index e4affc7f5396..93a3690fe1d3 100644
--- a/requirements/requirements-test.txt
+++ b/requirements/requirements-test.txt
@@ -1,4 +1,3 @@
-diffusers
 pytest
 coverage==7.2.3
 git+https://github.com/hpcaitech/pytest-testmon