4_mp_with_accelerate_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accelerate==0.34.2\n",
      "aiohappyeyeballs==2.4.0\n",
      "aiohttp==3.10.5\n",
      "aiosignal==1.3.1\n",
      "anyio==4.4.0\n",
      "argon2-cffi==23.1.0\n",
      "argon2-cffi-bindings==21.2.0\n",
      "arrow==1.3.0\n",
      "asttokens==2.4.1\n",
      "async-lru==2.0.4\n",
      "attrs==24.2.0\n",
      "babel==2.16.0\n",
      "beautifulsoup4==4.12.3\n",
      "bleach==6.1.0\n",
      "certifi==2024.8.30\n",
      "cffi==1.17.1\n",
      "charset-normalizer==3.3.2\n",
      "comm==0.2.2\n",
      "datasets==3.0.0\n",
      "debugpy==1.8.5\n",
      "decorator==5.1.1\n",
      "defusedxml==0.7.1\n",
      "dill==0.3.8\n",
      "executing==2.1.0\n",
      "fastjsonschema==2.20.0\n",
      "filelock==3.16.0\n",
      "fqdn==1.5.1\n",
      "frozenlist==1.4.1\n",
      "fsspec==2024.6.1\n",
      "h11==0.14.0\n",
      "httpcore==1.0.5\n",
      "httpx==0.27.2\n",
      "huggingface-hub==0.24.7\n",
      "idna==3.8\n",
      "ipykernel==6.29.5\n",
      "ipython==8.27.0\n",
      "ipywidgets==8.1.5\n",
      "isoduration==20.11.0\n",
      "jedi==0.19.1\n",
      "Jinja2==3.1.4\n",
      "json5==0.9.25\n",
      "jsonpointer==3.0.0\n",
      "jsonschema==4.23.0\n",
      "jsonschema-specifications==2023.12.1\n",
      "jupyter==1.1.1\n",
      "jupyter-console==6.6.3\n",
      "jupyter-events==0.10.0\n",
      "jupyter-lsp==2.2.5\n",
      "jupyter_client==8.6.2\n",
      "jupyter_core==5.7.2\n",
      "jupyter_server==2.14.2\n",
      "jupyter_server_terminals==0.5.3\n",
      "jupyterlab==4.2.5\n",
      "jupyterlab_pygments==0.3.0\n",
      "jupyterlab_server==2.27.3\n",
      "jupyterlab_widgets==3.0.13\n",
      "MarkupSafe==2.1.5\n",
      "matplotlib-inline==0.1.7\n",
      "mistune==3.0.2\n",
      "mpmath==1.3.0\n",
      "multidict==6.1.0\n",
      "multiprocess==0.70.16\n",
      "nbclient==0.10.0\n",
      "nbconvert==7.16.4\n",
      "nbformat==5.10.4\n",
      "nest-asyncio==1.6.0\n",
      "networkx==3.3\n",
      "notebook==7.2.2\n",
      "notebook_shim==0.2.4\n",
      "numpy==1.26.4\n",
      "nvidia-cublas-cu12==12.1.3.1\n",
      "nvidia-cuda-cupti-cu12==12.1.105\n",
      "nvidia-cuda-nvrtc-cu12==12.1.105\n",
      "nvidia-cuda-runtime-cu12==12.1.105\n",
      "nvidia-cudnn-cu12==8.9.2.26\n",
      "nvidia-cufft-cu12==11.0.2.54\n",
      "nvidia-curand-cu12==10.3.2.106\n",
      "nvidia-cusolver-cu12==11.4.5.107\n",
      "nvidia-cusparse-cu12==12.1.0.106\n",
      "nvidia-nccl-cu12==2.18.1\n",
      "nvidia-nvjitlink-cu12==12.6.68\n",
      "nvidia-nvtx-cu12==12.1.105\n",
      "overrides==7.7.0\n",
      "packaging==24.1\n",
      "pandas==2.2.2\n",
      "pandocfilters==1.5.1\n",
      "parso==0.8.4\n",
      "pexpect==4.9.0\n",
      "platformdirs==4.3.2\n",
      "prometheus_client==0.20.0\n",
      "prompt_toolkit==3.0.47\n",
      "psutil==6.0.0\n",
      "ptyprocess==0.7.0\n",
      "pure_eval==0.2.3\n",
      "pyarrow==17.0.0\n",
      "pycparser==2.22\n",
      "Pygments==2.18.0\n",
      "python-dateutil==2.9.0.post0\n",
      "python-json-logger==2.0.7\n",
      "pytz==2024.2\n",
      "PyYAML==6.0.2\n",
      "pyzmq==26.2.0\n",
      "referencing==0.35.1\n",
      "regex==2024.9.11\n",
      "requests==2.32.3\n",
      "rfc3339-validator==0.1.4\n",
      "rfc3986-validator==0.1.1\n",
      "rpds-py==0.20.0\n",
      "safetensors==0.4.5\n",
      "Send2Trash==1.8.3\n",
      "six==1.16.0\n",
      "sniffio==1.3.1\n",
      "soupsieve==2.6\n",
      "stack-data==0.6.3\n",
      "sympy==1.13.2\n",
      "terminado==0.18.1\n",
      "tinycss2==1.3.0\n",
      "tokenizers==0.19.1\n",
      "torch==2.1.0\n",
      "tornado==6.4.1\n",
      "tqdm==4.66.5\n",
      "traitlets==5.14.3\n",
      "transformers==4.44.2\n",
      "triton==2.1.0\n",
      "types-python-dateutil==2.9.0.20240906\n",
      "typing_extensions==4.12.2\n",
      "tzdata==2024.1\n",
      "uri-template==1.3.0\n",
      "urllib3==2.2.3\n",
      "wcwidth==0.2.13\n",
      "webcolors==24.8.0\n",
      "webencodings==0.5.1\n",
      "websocket-client==1.8.0\n",
      "widgetsnbextension==4.0.13\n",
      "xxhash==3.5.0\n",
      "yarl==1.11.1\n"
     ]
    }
   ],
   "source": [
    "!pip freeze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import transformers\n",
    "import torch\n",
    "\n",
    "model_id = \"meta-llama/Meta-Llama-3.1-8B-Instruct\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6ff635bd2e74e2d8b25e462e95e0cf1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipeline = transformers.pipeline(\n",
    "    \"text-generation\",\n",
    "    model=model_id,\n",
    "    model_kwargs={\"torch_dtype\": torch.float16},\n",
    "    device_map=\"auto\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model.embed_tokens': 0,\n",
       " 'model.layers.0': 0,\n",
       " 'model.layers.1': 1,\n",
       " 'model.layers.2': 1,\n",
       " 'model.layers.3': 1,\n",
       " 'model.layers.4': 1,\n",
       " 'model.layers.5': 1,\n",
       " 'model.layers.6': 2,\n",
       " 'model.layers.7': 2,\n",
       " 'model.layers.8': 2,\n",
       " 'model.layers.9': 2,\n",
       " 'model.layers.10': 2,\n",
       " 'model.layers.11': 3,\n",
       " 'model.layers.12': 3,\n",
       " 'model.layers.13': 3,\n",
       " 'model.layers.14': 3,\n",
       " 'model.layers.15': 3,\n",
       " 'model.layers.16': 4,\n",
       " 'model.layers.17': 4,\n",
       " 'model.layers.18': 4,\n",
       " 'model.layers.19': 4,\n",
       " 'model.layers.20': 4,\n",
       " 'model.layers.21': 5,\n",
       " 'model.layers.22': 5,\n",
       " 'model.layers.23': 5,\n",
       " 'model.layers.24': 5,\n",
       " 'model.layers.25': 5,\n",
       " 'model.layers.26': 6,\n",
       " 'model.layers.27': 6,\n",
       " 'model.layers.28': 6,\n",
       " 'model.layers.29': 6,\n",
       " 'model.layers.30': 6,\n",
       " 'model.layers.31': 7,\n",
       " 'model.norm': 7,\n",
       " 'model.rotary_emb': 7,\n",
       " 'lm_head': 7}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.model.hf_device_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}