NER命名实体识别demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# $$任务四：基于LSTM+CRF的序列标注$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.数据集预处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 加载数据，创建字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import os\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "from torchtext.vocab import GloVe\n",
    "import collections\n",
    "import os\n",
    "import random\n",
    "import time\n",
    "import torch.nn.functional as F\n",
    "from torch import nn\n",
    "import torchtext.vocab as Vocab\n",
    "import torch.utils.data as Data\n",
    "from tqdm import tqdm\n",
    "\n",
    "from nltk import word_tokenize\n",
    "import numpy as np\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "device=torch.device('cuda'if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path=r'E:\\NLP_jupyternotebook\\Fudan_NLP_beginner\\datasets\\CoNLL-2003'\n",
    "class CoNLL2003():\n",
    "    def read(self,data_path):\n",
    "        docs=['train','dev','test']\n",
    "        extension='.txt'\n",
    "        dataset={}\n",
    "        for doc in tqdm(docs):\n",
    "            doc_path=os.path.join(data_path,doc+extension)\n",
    "            dataset[doc]=self.read_file(str(doc_path))\n",
    "        return dataset\n",
    "    def read_file(self,doc_path):\n",
    "        samples=[]                              #目标形式[(['he','is','a','dog'],['o','o','o','o'])]\n",
    "        tokens=[]\n",
    "        tags=[]\n",
    "        with open(doc_path,'r',encoding='utf-8') as fb:\n",
    "            for line in fb:                     # example:\"briefing NN I-NP O\"\n",
    "                line=line.strip('\\n')\n",
    "                if line=='-DOCSTART- -X- -X- O': #去除每个txt文件的数据头\n",
    "                    pass\n",
    "                elif line=='':                  #观察文本中可以看到每段话是以一行空白分割开的\n",
    "                    if len(tokens)!=0:         #除了最开始的情况外，就是已经读取完了一句话，然后加入到samples里面\n",
    "                        samples.append((tokens,tags))\n",
    "                        tokens=[]\n",
    "                        tags=[]\n",
    "                else:\n",
    "                    contents=line.split(' ')\n",
    "                    tokens.append(contents[0])\n",
    "                    tags.append(contents[-1])\n",
    "        return samples   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:00<00:00,  9.78it/s]\n"
     ]
    }
   ],
   "source": [
    "conll2003=CoNLL2003()\n",
    "raw_data=conll2003.read(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------- 测试 ----------------------------------------\n",
      "(['EU', 'rejects', 'German', 'call', 'to', 'boycott', 'British', 'lamb', '.'], ['B-ORG', 'O', 'B-MISC', 'O', 'O', 'O', 'B-MISC', 'O', 'O'])\n",
      "(['Peter', 'Blackburn'], ['B-PER', 'I-PER'])\n",
      "(['BRUSSELS', '1996-08-22'], ['B-LOC', 'O'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "14041"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('-'*40,'测试','-'*40)\n",
    "for sample in raw_data['train'][:3]:\n",
    "    print(sample)\n",
    "len(raw_data['train'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_vocab(data):\n",
    "    counter=collections.Counter([tk for st,_ in data for tk in st])\n",
    "    return Vocab.Vocab(counter)\n",
    "def get_tag_vocab(data):\n",
    "    counter=collections.Counter([tk for _,st in data for tk in st])\n",
    "    return Vocab.Vocab(counter),counter\n",
    "vocab=get_vocab(raw_data['train']+raw_data['dev']+raw_data['test'])\n",
    "tag_vocab,tag_nums=get_tag_vocab(raw_data['train'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "label2id= {'<START>': 0, 'I-LOC': 1,'O': 2, 'B-LOC': 3, 'B-PER': 4, 'B-ORG': 5,'I-PER': 6,'I-ORG': 7,'B-MISC': 8,'I-MISC': 9,'<END>': 10}\n",
    "id2label=[item[0] for item in label2id.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------- 测试 ----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'<START>': 0,\n",
       " 'I-LOC': 1,\n",
       " 'O': 2,\n",
       " 'B-LOC': 3,\n",
       " 'B-PER': 4,\n",
       " 'B-ORG': 5,\n",
       " 'I-PER': 6,\n",
       " 'I-ORG': 7,\n",
       " 'B-MISC': 8,\n",
       " 'I-MISC': 9,\n",
       " '<END>': 10}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('-'*40,'测试','-'*40)\n",
    "label2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<START>',\n",
       " 'I-LOC',\n",
       " 'O',\n",
       " 'B-LOC',\n",
       " 'B-PER',\n",
       " 'B-ORG',\n",
       " 'I-PER',\n",
       " 'I-ORG',\n",
       " 'B-MISC',\n",
       " 'I-MISC',\n",
       " '<END>']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id2label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------- 测试 ----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "defaultdict(<bound method Vocab._default_unk_index of <torchtext.vocab.Vocab object at 0x000001BED0F24D08>>,\n",
       "            {'<unk>': 0,\n",
       "             '<pad>': 1,\n",
       "             ',': 2,\n",
       "             '.': 3,\n",
       "             'the': 4,\n",
       "             'of': 5,\n",
       "             'in': 6,\n",
       "             'to': 7,\n",
       "             'a': 8,\n",
       "             '(': 9,\n",
       "             ')': 10,\n",
       "             'and': 11,\n",
       "             '\"': 12,\n",
       "             'on': 13,\n",
       "             'said': 14,\n",
       "             \"'s\": 15,\n",
       "             'for': 16,\n",
       "             '-': 17,\n",
       "             '1': 18,\n",
       "             'The': 19,\n",
       "             'was': 20,\n",
       "             'at': 21,\n",
       "             '2': 22,\n",
       "             '3': 23,\n",
       "             '0': 24,\n",
       "             'with': 25,\n",
       "             'that': 26,\n",
       "             'from': 27,\n",
       "             'by': 28,\n",
       "             ':': 29,\n",
       "             'is': 30,\n",
       "             '4': 31,\n",
       "             'as': 32,\n",
       "             'he': 33,\n",
       "             'had': 34,\n",
       "             'his': 35,\n",
       "             'were': 36,\n",
       "             'it': 37,\n",
       "             'not': 38,\n",
       "             'has': 39,\n",
       "             'be': 40,\n",
       "             'an': 41,\n",
       "             'have': 42,\n",
       "             'after': 43,\n",
       "             'who': 44,\n",
       "             '5': 45,\n",
       "             'will': 46,\n",
       "             '6': 47,\n",
       "             'U.S.': 48,\n",
       "             'but': 49,\n",
       "             '$': 50,\n",
       "             'been': 51,\n",
       "             'first': 52,\n",
       "             'I': 53,\n",
       "             'are': 54,\n",
       "             'two': 55,\n",
       "             'which': 56,\n",
       "             'would': 57,\n",
       "             'their': 58,\n",
       "             '--': 59,\n",
       "             '7': 60,\n",
       "             'Friday': 61,\n",
       "             'beat': 62,\n",
       "             'up': 63,\n",
       "             'percent': 64,\n",
       "             'its': 65,\n",
       "             'they': 66,\n",
       "             'out': 67,\n",
       "             'this': 68,\n",
       "             'Thursday': 69,\n",
       "             'over': 70,\n",
       "             'one': 71,\n",
       "             'million': 72,\n",
       "             'year': 73,\n",
       "             'last': 74,\n",
       "             'against': 75,\n",
       "             'government': 76,\n",
       "             '10': 77,\n",
       "             '9': 78,\n",
       "             '/': 79,\n",
       "             '8': 80,\n",
       "             'Saturday': 81,\n",
       "             'A': 82,\n",
       "             'when': 83,\n",
       "             'police': 84,\n",
       "             'second': 85,\n",
       "             'He': 86,\n",
       "             'also': 87,\n",
       "             'It': 88,\n",
       "             'three': 89,\n",
       "             'Wednesday': 90,\n",
       "             'told': 91,\n",
       "             'about': 92,\n",
       "             'more': 93,\n",
       "             'or': 94,\n",
       "             'people': 95,\n",
       "             'new': 96,\n",
       "             '15': 97,\n",
       "             'into': 98,\n",
       "             'In': 99,\n",
       "             'no': 100,\n",
       "             'Germany': 101,\n",
       "             'won': 102,\n",
       "             'Tuesday': 103,\n",
       "             'week': 104,\n",
       "             'New': 105,\n",
       "             'But': 106,\n",
       "             'market': 107,\n",
       "             \"'\": 108,\n",
       "             'under': 109,\n",
       "             '11': 110,\n",
       "             'we': 111,\n",
       "             '13': 112,\n",
       "             'group': 113,\n",
       "             'than': 114,\n",
       "             'SOCCER': 115,\n",
       "             'some': 116,\n",
       "             'AT': 117,\n",
       "             'Monday': 118,\n",
       "             'before': 119,\n",
       "             'match': 120,\n",
       "             'could': 121,\n",
       "             'points': 122,\n",
       "             'Australia': 123,\n",
       "             'France': 124,\n",
       "             'between': 125,\n",
       "             'time': 126,\n",
       "             '20': 127,\n",
       "             '12': 128,\n",
       "             '14': 129,\n",
       "             'her': 130,\n",
       "             'years': 131,\n",
       "             'We': 132,\n",
       "             'other': 133,\n",
       "             'since': 134,\n",
       "             '21': 135,\n",
       "             'all': 136,\n",
       "             'only': 137,\n",
       "             'played': 138,\n",
       "             'there': 139,\n",
       "             '22': 140,\n",
       "             'Cup': 141,\n",
       "             'England': 142,\n",
       "             'South': 143,\n",
       "             'Sunday': 144,\n",
       "             'off': 145,\n",
       "             'round': 146,\n",
       "             'down': 147,\n",
       "             'him': 148,\n",
       "             'officials': 149,\n",
       "             'next': 150,\n",
       "             'four': 151,\n",
       "             'United': 152,\n",
       "             '16': 153,\n",
       "             'NEW': 154,\n",
       "             '1996-08-28': 155,\n",
       "             'Russia': 156,\n",
       "             'Britain': 157,\n",
       "             'President': 158,\n",
       "             '1996-12-06': 159,\n",
       "             'China': 160,\n",
       "             'third': 161,\n",
       "             'home': 162,\n",
       "             'six': 163,\n",
       "             'Minister': 164,\n",
       "             'five': 165,\n",
       "             'lost': 166,\n",
       "             'Italy': 167,\n",
       "             'spokesman': 168,\n",
       "             'state': 169,\n",
       "             'company': 170,\n",
       "             'July': 171,\n",
       "             'day': 172,\n",
       "             'did': 173,\n",
       "             'games': 174,\n",
       "             'if': 175,\n",
       "             'World': 176,\n",
       "             'because': 177,\n",
       "             'win': 178,\n",
       "             'made': 179,\n",
       "             \"n't\": 180,\n",
       "             'LONDON': 181,\n",
       "             'official': 182,\n",
       "             '70': 183,\n",
       "             'Spain': 184,\n",
       "             'any': 185,\n",
       "             'former': 186,\n",
       "             '1996': 187,\n",
       "             'September': 188,\n",
       "             'YORK': 189,\n",
       "             'them': 190,\n",
       "             'world': 191,\n",
       "             'DIVISION': 192,\n",
       "             'back': 193,\n",
       "             'do': 194,\n",
       "             '30': 195,\n",
       "             'RESULTS': 196,\n",
       "             'she': 197,\n",
       "             'where': 198,\n",
       "             '3.': 199,\n",
       "             '6-4': 200,\n",
       "             '1.': 201,\n",
       "             '17': 202,\n",
       "             '2.': 203,\n",
       "             'Japan': 204,\n",
       "             'expected': 205,\n",
       "             'through': 206,\n",
       "             'just': 207,\n",
       "             'being': 208,\n",
       "             'matches': 209,\n",
       "             'v': 210,\n",
       "             '6-3': 211,\n",
       "             'They': 212,\n",
       "             'team': 213,\n",
       "             '1/2': 214,\n",
       "             'European': 215,\n",
       "             'statement': 216,\n",
       "             ';': 217,\n",
       "             'b': 218,\n",
       "             'meeting': 219,\n",
       "             'talks': 220,\n",
       "             '69': 221,\n",
       "             '25': 222,\n",
       "             'Russian': 223,\n",
       "             'news': 224,\n",
       "             'while': 225,\n",
       "             '1996-08-29': 226,\n",
       "             'peace': 227,\n",
       "             '71': 228,\n",
       "             'August': 229,\n",
       "             'British': 230,\n",
       "             'killed': 231,\n",
       "             'June': 232,\n",
       "             'added': 233,\n",
       "             'during': 234,\n",
       "             '1996-08-22': 235,\n",
       "             'game': 236,\n",
       "             'London': 237,\n",
       "             'set': 238,\n",
       "             'German': 239,\n",
       "             'president': 240,\n",
       "             'Clinton': 241,\n",
       "             'National': 242,\n",
       "             'Results': 243,\n",
       "             'billion': 244,\n",
       "             'reported': 245,\n",
       "             '1996-08-27': 246,\n",
       "             '6-2': 247,\n",
       "             'Reuters': 248,\n",
       "             'take': 249,\n",
       "             '24': 250,\n",
       "             'month': 251,\n",
       "             'now': 252,\n",
       "             'division': 253,\n",
       "             'seconds': 254,\n",
       "             'shares': 255,\n",
       "             'lead': 256,\n",
       "             'metres': 257,\n",
       "             'saying': 258,\n",
       "             'took': 259,\n",
       "             'minutes': 260,\n",
       "             'most': 261,\n",
       "             '18': 262,\n",
       "             'end': 263,\n",
       "             'West': 264,\n",
       "             'international': 265,\n",
       "             'soccer': 266,\n",
       "             'half': 267,\n",
       "             'season': 268,\n",
       "             'still': 269,\n",
       "             '1996-08-26': 270,\n",
       "             'Israel': 271,\n",
       "             'party': 272,\n",
       "             '1996-08-31': 273,\n",
       "             'political': 274,\n",
       "             'York': 275,\n",
       "             'Sweden': 276,\n",
       "             'earlier': 277,\n",
       "             'held': 278,\n",
       "             'innings': 279,\n",
       "             'should': 280,\n",
       "             'tonnes': 281,\n",
       "             '1996-08-30': 282,\n",
       "             'At': 283,\n",
       "             '4.': 284,\n",
       "             'part': 285,\n",
       "             '5.': 286,\n",
       "             'American': 287,\n",
       "             'Pakistan': 288,\n",
       "             'close': 289,\n",
       "             'early': 290,\n",
       "             'final': 291,\n",
       "             'city': 292,\n",
       "             'days': 293,\n",
       "             'our': 294,\n",
       "             '1996-08-23': 295,\n",
       "             'minute': 296,\n",
       "             '6.': 297,\n",
       "             'capital': 298,\n",
       "             'country': 299,\n",
       "             'due': 300,\n",
       "             'lower': 301,\n",
       "             '100': 302,\n",
       "             'Bank': 303,\n",
       "             'deal': 304,\n",
       "             '72': 305,\n",
       "             'left': 306,\n",
       "             'play': 307,\n",
       "             'so': 308,\n",
       "             'foreign': 309,\n",
       "             'Iraq': 310,\n",
       "             'can': 311,\n",
       "             'number': 312,\n",
       "             'then': 313,\n",
       "             'victory': 314,\n",
       "             'French': 315,\n",
       "             'Open': 316,\n",
       "             'around': 317,\n",
       "             'seven': 318,\n",
       "             '28': 319,\n",
       "             'league': 320,\n",
       "             'prices': 321,\n",
       "             'run': 322,\n",
       "             'well': 323,\n",
       "             '23': 324,\n",
       "             '68': 325,\n",
       "             'Belgium': 326,\n",
       "             'India': 327,\n",
       "             'trade': 328,\n",
       "             'goals': 329,\n",
       "             'results': 330,\n",
       "             '75': 331,\n",
       "             'Iraqi': 332,\n",
       "             'Newsroom': 333,\n",
       "             'cents': 334,\n",
       "             'war': 335,\n",
       "             'Czech': 336,\n",
       "             'leader': 337,\n",
       "             'military': 338,\n",
       "             'per': 339,\n",
       "             '1995': 340,\n",
       "             '67': 341,\n",
       "             'May': 342,\n",
       "             'Netherlands': 343,\n",
       "             'found': 344,\n",
       "             'says': 345,\n",
       "             'very': 346,\n",
       "             'States': 347,\n",
       "             'called': 348,\n",
       "             'elections': 349,\n",
       "             'visit': 350,\n",
       "             '26': 351,\n",
       "             '50': 352,\n",
       "             'League': 353,\n",
       "             'make': 354,\n",
       "             '7.': 355,\n",
       "             'Africa': 356,\n",
       "             'IN': 357,\n",
       "             'say': 358,\n",
       "             'town': 359,\n",
       "             '19': 360,\n",
       "             'LEAGUE': 361,\n",
       "             'Moscow': 362,\n",
       "             'eight': 363,\n",
       "             'meet': 364,\n",
       "             'CHICAGO': 365,\n",
       "             'Republic': 366,\n",
       "             'ago': 367,\n",
       "             'man': 368,\n",
       "             'rate': 369,\n",
       "             '1996-08-25': 370,\n",
       "             '8.': 371,\n",
       "             'Michael': 372,\n",
       "             'both': 373,\n",
       "             'champion': 374,\n",
       "             'go': 375,\n",
       "             'good': 376,\n",
       "             'newspaper': 377,\n",
       "             'northern': 378,\n",
       "             'same': 379,\n",
       "             'Canada': 380,\n",
       "             'gave': 381,\n",
       "             'local': 382,\n",
       "             'months': 383,\n",
       "             'place': 384,\n",
       "             'agency': 385,\n",
       "             'my': 386,\n",
       "             'put': 387,\n",
       "             'troops': 388,\n",
       "             '6-1': 389,\n",
       "             '73': 390,\n",
       "             'Mark': 391,\n",
       "             'ended': 392,\n",
       "             'rebels': 393,\n",
       "             'runs': 394,\n",
       "             'start': 395,\n",
       "             'what': 396,\n",
       "             '29': 397,\n",
       "             '7-6': 398,\n",
       "             'Israeli': 399,\n",
       "             'dollar': 400,\n",
       "             'price': 401,\n",
       "             '31': 402,\n",
       "             'Attendance': 403,\n",
       "             'Men': 404,\n",
       "             'Women': 405,\n",
       "             'behind': 406,\n",
       "             'issue': 407,\n",
       "             'sales': 408,\n",
       "             'you': 409,\n",
       "             '27': 410,\n",
       "             'Austria': 411,\n",
       "             'hit': 412,\n",
       "             'late': 413,\n",
       "             'players': 414,\n",
       "             'think': 415,\n",
       "             'David': 416,\n",
       "             'There': 417,\n",
       "             'another': 418,\n",
       "             'bank': 419,\n",
       "             'newsroom': 420,\n",
       "             'profit': 421,\n",
       "             'quoted': 422,\n",
       "             'report': 423,\n",
       "             '66': 424,\n",
       "             'Chicago': 425,\n",
       "             'Prime': 426,\n",
       "             'forces': 427,\n",
       "             'leaders': 428,\n",
       "             'minister': 429,\n",
       "             'race': 430,\n",
       "             'singles': 431,\n",
       "             'until': 432,\n",
       "             '74': 433,\n",
       "             'championship': 434,\n",
       "             'may': 435,\n",
       "             'security': 436,\n",
       "             'support': 437,\n",
       "             '64': 438,\n",
       "             '65': 439,\n",
       "             'African': 440,\n",
       "             'Dutch': 441,\n",
       "             'Inc': 442,\n",
       "             'St': 443,\n",
       "             'closed': 444,\n",
       "             'going': 445,\n",
       "             'men': 446,\n",
       "             'top': 447,\n",
       "             'tournament': 448,\n",
       "             '60': 449,\n",
       "             'Australian': 450,\n",
       "             'Hong': 451,\n",
       "             'Yeltsin': 452,\n",
       "             'court': 453,\n",
       "             'economic': 454,\n",
       "             'fell': 455,\n",
       "             'head': 456,\n",
       "             'including': 457,\n",
       "             'opposition': 458,\n",
       "             'side': 459,\n",
       "             'CRICKET': 460,\n",
       "             'John': 461,\n",
       "             'many': 462,\n",
       "             'want': 463,\n",
       "             'Paul': 464,\n",
       "             'area': 465,\n",
       "             'authorities': 466,\n",
       "             'came': 467,\n",
       "             'following': 468,\n",
       "             'later': 469,\n",
       "             'miles': 470,\n",
       "             'record': 471,\n",
       "             '7-5': 472,\n",
       "             'U.N.': 473,\n",
       "             'arrested': 474,\n",
       "             'get': 475,\n",
       "             'office': 476,\n",
       "             '1-0': 477,\n",
       "             'SAN': 478,\n",
       "             'ahead': 479,\n",
       "             'allowed': 480,\n",
       "             'drawn': 481,\n",
       "             'parliament': 482,\n",
       "             'refugees': 483,\n",
       "             'women': 484,\n",
       "             'East': 485,\n",
       "             'Palestinian': 486,\n",
       "             'Party': 487,\n",
       "             '...': 488,\n",
       "             'Kong': 489,\n",
       "             'OF': 490,\n",
       "             'c': 491,\n",
       "             'central': 492,\n",
       "             'km': 493,\n",
       "             'near': 494,\n",
       "             'plan': 495,\n",
       "             'region': 496,\n",
       "             'several': 497,\n",
       "             'taking': 498,\n",
       "             '40': 499,\n",
       "             'Brazil': 500,\n",
       "             'FIRST': 501,\n",
       "             'agreement': 502,\n",
       "             'already': 503,\n",
       "             'chief': 504,\n",
       "             'like': 505,\n",
       "             'own': 506,\n",
       "             'weekend': 507,\n",
       "             '1996-08-24': 508,\n",
       "             'Arafat': 509,\n",
       "             'Foreign': 510,\n",
       "             'national': 511,\n",
       "             'those': 512,\n",
       "             'Lebed': 513,\n",
       "             'M.': 514,\n",
       "             'Police': 515,\n",
       "             'Two': 516,\n",
       "             'further': 517,\n",
       "             'give': 518,\n",
       "             'net': 519,\n",
       "             'right': 520,\n",
       "             'test': 521,\n",
       "             'trading': 522,\n",
       "             'English': 523,\n",
       "             'GMT': 524,\n",
       "             'Ireland': 525,\n",
       "             'Zealand': 526,\n",
       "             'away': 527,\n",
       "             'demand': 528,\n",
       "             'general': 529,\n",
       "             'me': 530,\n",
       "             'ministry': 531,\n",
       "             'plans': 532,\n",
       "             'vs.': 533,\n",
       "             'work': 534,\n",
       "             '1994': 535,\n",
       "             'Democratic': 536,\n",
       "             'This': 537,\n",
       "             'agreed': 538,\n",
       "             'decision': 539,\n",
       "             'fighting': 540,\n",
       "             'higher': 541,\n",
       "             'loss': 542,\n",
       "             'shot': 543,\n",
       "             'way': 544,\n",
       "             '1997': 545,\n",
       "             'Chechnya': 546,\n",
       "             'STANDINGS': 547,\n",
       "             'much': 548,\n",
       "             'CUP': 549,\n",
       "             'City': 550,\n",
       "             'Indonesia': 551,\n",
       "             'Union': 552,\n",
       "             'announced': 553,\n",
       "             'best': 554,\n",
       "             'conference': 555,\n",
       "             'election': 556,\n",
       "             'hours': 557,\n",
       "             'night': 558,\n",
       "             'oil': 559,\n",
       "             'reporters': 560,\n",
       "             'return': 561,\n",
       "             'seen': 562,\n",
       "             'taken': 563,\n",
       "             'weeks': 564,\n",
       "             'Ahmed': 565,\n",
       "             'Corp': 566,\n",
       "             'L': 567,\n",
       "             'Poland': 568,\n",
       "             'asked': 569,\n",
       "             'campaign': 570,\n",
       "             'high': 571,\n",
       "             'money': 572,\n",
       "             'overs': 573,\n",
       "             'strong': 574,\n",
       "             'whether': 575,\n",
       "             'without': 576,\n",
       "             'Standings': 577,\n",
       "             'army': 578,\n",
       "             'attack': 579,\n",
       "             'major': 580,\n",
       "             'nine': 581,\n",
       "             'such': 582,\n",
       "             '2-0': 583,\n",
       "             'December': 584,\n",
       "             'Dole': 585,\n",
       "             'Martin': 586,\n",
       "             'Sri': 587,\n",
       "             'began': 588,\n",
       "             'captain': 589,\n",
       "             'de': 590,\n",
       "             'long': 591,\n",
       "             'rose': 592,\n",
       "             'scored': 593,\n",
       "             'southern': 594,\n",
       "             '9.': 595,\n",
       "             'Washington': 596,\n",
       "             'bond': 597,\n",
       "             'control': 598,\n",
       "             'died': 599,\n",
       "             'halftime': 600,\n",
       "             'past': 601,\n",
       "             'pct': 602,\n",
       "             'winning': 603,\n",
       "             \"'S\": 604,\n",
       "             'Europe': 605,\n",
       "             'Italian': 606,\n",
       "             'airport': 607,\n",
       "             'future': 608,\n",
       "             'hold': 609,\n",
       "             'index': 610,\n",
       "             'leading': 611,\n",
       "             'quarter': 612,\n",
       "             'went': 613,\n",
       "             'wickets': 614,\n",
       "             'Commission': 615,\n",
       "             'Jerusalem': 616,\n",
       "             'On': 617,\n",
       "             'club': 618,\n",
       "             'come': 619,\n",
       "             'does': 620,\n",
       "             'few': 621,\n",
       "             'markets': 622,\n",
       "             'north': 623,\n",
       "             'released': 624,\n",
       "             'rights': 625,\n",
       "             'sent': 626,\n",
       "             'share': 627,\n",
       "             'stock': 628,\n",
       "             'television': 629,\n",
       "             'children': 630,\n",
       "             'led': 631,\n",
       "             'main': 632,\n",
       "             'manager': 633,\n",
       "             'Kurdish': 634,\n",
       "             'coach': 635,\n",
       "             'countries': 636,\n",
       "             'interest': 637,\n",
       "             'members': 638,\n",
       "             'plane': 639,\n",
       "             'power': 640,\n",
       "             'rates': 641,\n",
       "             'signed': 642,\n",
       "             'tour': 643,\n",
       "             'us': 644,\n",
       "             'yen': 645,\n",
       "             '63': 646,\n",
       "             'An': 647,\n",
       "             'Co': 648,\n",
       "             'Finland': 649,\n",
       "             'One': 650,\n",
       "             'W': 651,\n",
       "             'bonds': 652,\n",
       "             'business': 653,\n",
       "             'case': 654,\n",
       "             'details': 655,\n",
       "             'growth': 656,\n",
       "             'opening': 657,\n",
       "             'order': 658,\n",
       "             'tabulate': 659,\n",
       "             '1996-12-07': 660,\n",
       "             'April': 661,\n",
       "             'Group': 662,\n",
       "             'March': 663,\n",
       "             'PCT': 664,\n",
       "             'Switzerland': 665,\n",
       "             'death': 666,\n",
       "             'hits': 667,\n",
       "             'might': 668,\n",
       "             'morning': 669,\n",
       "             'these': 670,\n",
       "             '10.': 671,\n",
       "             'If': 672,\n",
       "             'Moslem': 673,\n",
       "             'Olympic': 674,\n",
       "             'State': 675,\n",
       "             'again': 676,\n",
       "             'available': 677,\n",
       "             'average': 678,\n",
       "             'help': 679,\n",
       "             'possible': 680,\n",
       "             'production': 681,\n",
       "             'recent': 682,\n",
       "             'title': 683,\n",
       "             'total': 684,\n",
       "             'yet': 685,\n",
       "             '96': 686,\n",
       "             'Jordan': 687,\n",
       "             'October': 688,\n",
       "             'Peter': 689,\n",
       "             'WORLD': 690,\n",
       "             'comment': 691,\n",
       "             'previous': 692,\n",
       "             'accused': 693,\n",
       "             'air': 694,\n",
       "             'cash': 695,\n",
       "             'current': 696,\n",
       "             'draw': 697,\n",
       "             'fourth': 698,\n",
       "             'immediately': 699,\n",
       "             'must': 700,\n",
       "             'outside': 701,\n",
       "             'period': 702,\n",
       "             'reports': 703,\n",
       "             'strike': 704,\n",
       "             '62': 705,\n",
       "             'Division': 706,\n",
       "             'Leading': 707,\n",
       "             'Mexico': 708,\n",
       "             'Romania': 709,\n",
       "             'Turkey': 710,\n",
       "             'ban': 711,\n",
       "             'centre': 712,\n",
       "             'failed': 713,\n",
       "             'got': 714,\n",
       "             'known': 715,\n",
       "             'law': 716,\n",
       "             'least': 717,\n",
       "             'likely': 718,\n",
       "             'making': 719,\n",
       "             'standings': 720,\n",
       "             'too': 721,\n",
       "             'vs': 722,\n",
       "             '32': 723,\n",
       "             '54': 724,\n",
       "             '76': 725,\n",
       "             'budget': 726,\n",
       "             'conditions': 727,\n",
       "             'cut': 728,\n",
       "             'despite': 729,\n",
       "             'free': 730,\n",
       "             'given': 731,\n",
       "             'goal': 732,\n",
       "             'policy': 733,\n",
       "             'press': 734,\n",
       "             'see': 735,\n",
       "             'series': 736,\n",
       "             'small': 737,\n",
       "             'started': 738,\n",
       "             'students': 739,\n",
       "             'trying': 740,\n",
       "             '61': 741,\n",
       "             'First': 742,\n",
       "             'Lanka': 743,\n",
       "             'Paris': 744,\n",
       "             'Polish': 745,\n",
       "             'Scotland': 746,\n",
       "             'White': 747,\n",
       "             'contract': 748,\n",
       "             'date': 749,\n",
       "             'each': 750,\n",
       "             'economy': 751,\n",
       "             'forced': 752,\n",
       "             'prison': 753,\n",
       "             'public': 754,\n",
       "             'rise': 755,\n",
       "             'settlement': 756,\n",
       "             'south': 757,\n",
       "             'term': 758,\n",
       "             'used': 759,\n",
       "             'working': 760,\n",
       "             '33': 761,\n",
       "             '59': 762,\n",
       "             'CITY': 763,\n",
       "             'International': 764,\n",
       "             'Kenya': 765,\n",
       "             'Ministry': 766,\n",
       "             'No': 767,\n",
       "             'S.': 768,\n",
       "             'Scorers': 769,\n",
       "             'Singapore': 770,\n",
       "             'Sydney': 771,\n",
       "             'TO': 772,\n",
       "             'Ukraine': 773,\n",
       "             'bid': 774,\n",
       "             'champions': 775,\n",
       "             'charges': 776,\n",
       "             'hospital': 777,\n",
       "             'injured': 778,\n",
       "             'little': 779,\n",
       "             'point': 780,\n",
       "             'position': 781,\n",
       "             'A.': 782,\n",
       "             'Belgian': 783,\n",
       "             'Canadian': 784,\n",
       "             'Costa': 785,\n",
       "             'January': 786,\n",
       "             'Nations': 787,\n",
       "             'Nigeria': 788,\n",
       "             'Norway': 789,\n",
       "             'Robert': 790,\n",
       "             'San': 791,\n",
       "             'That': 792,\n",
       "             'Thomas': 793,\n",
       "             'Wasim': 794,\n",
       "             'Wimbledon': 795,\n",
       "             'estimated': 796,\n",
       "             'fifth': 797,\n",
       "             'great': 798,\n",
       "             'house': 799,\n",
       "             'keep': 800,\n",
       "             'move': 801,\n",
       "             'open': 802,\n",
       "             'penalty': 803,\n",
       "             'prime': 804,\n",
       "             'within': 805,\n",
       "             'Akram': 806,\n",
       "             'BEAT': 807,\n",
       "             'Ltd': 808,\n",
       "             'action': 809,\n",
       "             'analysts': 810,\n",
       "             'clear': 811,\n",
       "             'even': 812,\n",
       "             'groups': 813,\n",
       "             'pay': 814,\n",
       "             'planned': 815,\n",
       "             'radio': 816,\n",
       "             'rebel': 817,\n",
       "             'received': 818,\n",
       "             'tennis': 819,\n",
       "             'today': 820,\n",
       "             'unless': 821,\n",
       "             'use': 822,\n",
       "             'vote': 823,\n",
       "             'workers': 824,\n",
       "             \"'m\": 825,\n",
       "             '0-0': 826,\n",
       "             '1,000': 827,\n",
       "             '34': 828,\n",
       "             '77': 829,\n",
       "             'Barcelona': 830,\n",
       "             'Department': 831,\n",
       "             'Iran': 832,\n",
       "             'Jones': 833,\n",
       "             'Mushtaq': 834,\n",
       "             'North': 835,\n",
       "             'TORONTO': 836,\n",
       "             'Total': 837,\n",
       "             'brought': 838,\n",
       "             'call': 839,\n",
       "             'companies': 840,\n",
       "             'face': 841,\n",
       "             'force': 842,\n",
       "             'forecast': 843,\n",
       "             'gold': 844,\n",
       "             'industry': 845,\n",
       "             'level': 846,\n",
       "             'never': 847,\n",
       "             'protest': 848,\n",
       "             'road': 849,\n",
       "             'service': 850,\n",
       "             'straight': 851,\n",
       "             'whose': 852,\n",
       "             'woman': 853,\n",
       "             \"'re\": 854,\n",
       "             '35': 855,\n",
       "             'BASEBALL': 856,\n",
       "             'Exchange': 857,\n",
       "             'Swiss': 858,\n",
       "             'across': 859,\n",
       "             'better': 860,\n",
       "             'ceasefire': 861,\n",
       "             'declared': 862,\n",
       "             'embassy': 863,\n",
       "             'firm': 864,\n",
       "             'homer': 865,\n",
       "             'know': 866,\n",
       "             'low': 867,\n",
       "             'result': 868,\n",
       "             '2-1': 869,\n",
       "             '45': 870,\n",
       "             '46': 871,\n",
       "             '48': 872,\n",
       "             'After': 873,\n",
       "             'Chinese': 874,\n",
       "             'PARIS': 875,\n",
       "             'Security': 876,\n",
       "             'Slovakia': 877,\n",
       "             'Van': 878,\n",
       "             'based': 879,\n",
       "             'charged': 880,\n",
       "             'daily': 881,\n",
       "             'disease': 882,\n",
       "             'financial': 883,\n",
       "             'himself': 884,\n",
       "             'nearly': 885,\n",
       "             'services': 886,\n",
       "             'showed': 887,\n",
       "             'squad': 888,\n",
       "             'stories': 889,\n",
       "             'striker': 890,\n",
       "             'violence': 891,\n",
       "             '&': 892,\n",
       "             '36': 893,\n",
       "             'Amsterdam': 894,\n",
       "             'Association': 895,\n",
       "             'Aug': 896,\n",
       "             'Bosnia': 897,\n",
       "             'Bosnian': 898,\n",
       "             'California': 899,\n",
       "             'Council': 900,\n",
       "             'Dutroux': 901,\n",
       "             'Ian': 902,\n",
       "             'Madrid': 903,\n",
       "             'Major': 904,\n",
       "             'November': 905,\n",
       "             'Result': 906,\n",
       "             'She': 907,\n",
       "             'TENNIS': 908,\n",
       "             'attacks': 909,\n",
       "             'coming': 910,\n",
       "             'here': 911,\n",
       "             'old': 912,\n",
       "             'process': 913,\n",
       "             'qualifier': 914,\n",
       "             'reached': 915,\n",
       "             'scheduled': 916,\n",
       "             'short': 917,\n",
       "             'trip': 918,\n",
       "             'village': 919,\n",
       "             '---': 920,\n",
       "             '1992': 921,\n",
       "             '42': 922,\n",
       "             '53': 923,\n",
       "             '6-0': 924,\n",
       "             'Ajax': 925,\n",
       "             'Atlanta': 926,\n",
       "             'Indian': 927,\n",
       "             'Korea': 928,\n",
       "             'Republican': 929,\n",
       "             'Series': 930,\n",
       "             'WASHINGTON': 931,\n",
       "             'confirmed': 932,\n",
       "             'independence': 933,\n",
       "             'latest': 934,\n",
       "             'negotiations': 935,\n",
       "             'offered': 936,\n",
       "             'senior': 937,\n",
       "             'struck': 938,\n",
       "             'tax': 939,\n",
       "             'turned': 940,\n",
       "             '1993': 941,\n",
       "             '55': 942,\n",
       "             '56': 943,\n",
       "             'BOSTON': 944,\n",
       "             'Central': 945,\n",
       "             'DETROIT': 946,\n",
       "             'Halftime': 947,\n",
       "             'Khan': 948,\n",
       "             'Net': 949,\n",
       "             'Netanyahu': 950,\n",
       "             'OSCE': 951,\n",
       "             'Wall': 952,\n",
       "             'Waqar': 953,\n",
       "             'bomb': 954,\n",
       "             'border': 955,\n",
       "             'chairman': 956,\n",
       "             'decided': 957,\n",
       "             'declined': 958,\n",
       "             'defeat': 959,\n",
       "             'denied': 960,\n",
       "             'earnings': 961,\n",
       "             'eastern': 962,\n",
       "             'exports': 963,\n",
       "             'fire': 964,\n",
       "             'guerrillas': 965,\n",
       "             'how': 966,\n",
       "             'human': 967,\n",
       "             'illegal': 968,\n",
       "             'life': 969,\n",
       "             'named': 970,\n",
       "             'need': 971,\n",
       "             'opened': 972,\n",
       "             'problem': 973,\n",
       "             'province': 974,\n",
       "             'really': 975,\n",
       "             'soon': 976,\n",
       "             'source': 977,\n",
       "             'stocks': 978,\n",
       "             '1-1': 979,\n",
       "             '200': 980,\n",
       "             '81': 981,\n",
       "             'All': 982,\n",
       "             'FOR': 983,\n",
       "             'Grand': 984,\n",
       "             'Japanese': 985,\n",
       "             'OPEN': 986,\n",
       "             'ST': 987,\n",
       "             'Street': 988,\n",
       "             'areas': 989,\n",
       "             'believed': 990,\n",
       "             'career': 991,\n",
       "             'continue': 992,\n",
       "             'course': 993,\n",
       "             'full': 994,\n",
       "             'having': 995,\n",
       "             'met': 996,\n",
       "             'offer': 997,\n",
       "             'ordered': 998,\n",
       "             'passengers': 999,\n",
       "             ...})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('-'*40,'测试','-'*40)\n",
    "vocab.stoi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30291, 30291)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab.itos),len(vocab.stoi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 建立数据迭代器：每次迭代的数据包含训练集[batch，seq_len]，标签[batch，seq_len]，真实单词数[batch]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先创建自定义的数据类，然后定制collate_fn设置动态padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.utils.data import DataLoader\n",
    "class MyDataSet(Dataset):\n",
    "    def __init__(self,raw_data,vocab,label2id):\n",
    "        self.raw_data=raw_data\n",
    "        self.vocab=vocab.stoi\n",
    "        self.label2id=label2id\n",
    "        self.x_list=[[vocab.stoi[w] if w in vocab.stoi else vocab.stoi['<unk>'] for w in item[0]]for item in raw_data]\n",
    "        self.y_list=[[label2id[lable]for lable in item[1]]for item in raw_data]\n",
    "    def __getitem__(self,item):\n",
    "        return torch.Tensor(self.x_list[item]),torch.Tensor(self.y_list[item])\n",
    "    def __len__(self):\n",
    "        return len(self.raw_data)\n",
    "def collate_fn(batch):\n",
    "    x_list=[x[0]for x in batch]\n",
    "    y_list=[x[1]for x in batch]\n",
    "    lengths=[len(item[0])for item in batch]\n",
    "    x_list=pad_sequence(x_list,padding_value=vocab.stoi['<pad>'])\n",
    "    y_list=pad_sequence(y_list,padding_value=-1)\n",
    "    return x_list.long().transpose(0,1),y_list.long().transpose(0,1),lengths\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set=MyDataSet(raw_data['train'],vocab,label2id)\n",
    "test_set=MyDataSet(raw_data['test'],vocab,label2id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "查看数据类和数据迭代器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------- 测试 ----------------------------------------\n",
      "原始数据：\n",
      " (['EU', 'rejects', 'German', 'call', 'to', 'boycott', 'British', 'lamb', '.'], ['B-ORG', 'O', 'B-MISC', 'O', 'O', 'O', 'B-MISC', 'O', 'O'])\n",
      "自定义数据：\n",
      " (tensor([1.0490e+03, 1.4971e+04, 2.3900e+02, 8.3900e+02, 7.0000e+00, 4.0960e+03,\n",
      "        2.3000e+02, 1.0093e+04, 3.0000e+00]), tensor([5., 2., 8., 2., 2., 2., 8., 2., 2.])) ['EU', 'rejects', 'German', 'call', 'to', 'boycott', 'British', 'lamb', '.']\n"
     ]
    }
   ],
   "source": [
    "def id2sentence(sequence,vocab):\n",
    "    return [vocab.itos[int(w)]for w in sequence]\n",
    "print('-'*40,'测试','-'*40)\n",
    "print('原始数据：\\n',raw_data['train'][0])\n",
    "print('自定义数据：\\n',train_set[0],id2sentence(train_set[0][0],vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------- 测试 ----------------------------------------\n",
      "0 th\t训练集batch: tensor([[ 1049, 14971,   239,   839,     7,  4096,   230, 10093,     3],\n",
      "        [  689,  2077,     1,     1,     1,     1,     1,     1,     1]]) torch.Size([2, 9]) \n",
      "\t标签 batch: torch.Size([2, 9]) \n",
      "\t真实单词数： [9, 2]\n"
     ]
    }
   ],
   "source": [
    "train_iter=DataLoader(dataset=train_set,batch_size=2,collate_fn=collate_fn)\n",
    "test_iter=DataLoader(dataset=test_set,batch_size=2,collate_fn=collate_fn)\n",
    "print('-'*40,'测试','-'*40)\n",
    "for idx,x in enumerate(train_iter):\n",
    "    print(idx,'th\\t训练集batch:',x[0],x[0].shape,\"\\n\\t标签 batch:\",x[1].shape,\"\\n\\t真实单词数：\",x[2])\n",
    "    #if idx==4:\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mask(lengths):\n",
    "    \"\"\"\n",
    "    Parameters:\n",
    "        lengths--由数据迭代器返回的第三个数据，包含了一个batch中的每个样本的真实单词数，也就是上面的真实单词数 list of int\n",
    "    Return:\n",
    "        mask-- 由Lengths生成的掩码矩阵，在真实单词上为1，填充的地方为0\n",
    "    \"\"\"\n",
    "    max_len=max(lengths)\n",
    "    mask=torch.Tensor()\n",
    "    for length in lengths:\n",
    "        mask=torch.cat((mask,torch.Tensor([[1]*length+[0]*(max_len-length)])),dim=0)\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------- 测试 ----------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 0., 0.],\n",
       "        [1., 1., 1., 0.],\n",
       "        [1., 1., 1., 1.]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('-'*40,'测试','-'*40)\n",
    "a=[2,3,4]\n",
    "b=get_mask(a)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30291"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab.stoi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.建立模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "模型分为两个大部分，第一个是LSTM模型，目的是通过输出数据得到发射矩阵；第二个是CRF模型，目的是通过已知的观测序列进行参数学习得到最优的参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "from torch.nn import init\n",
    "from torch.nn.utils.rnn import pack_padded_sequence,pad_packed_sequence\n",
    "\n",
    "class LstmCrf(nn.Module):\n",
    "    def __init__(self,input_dim,hidden_dim,label2id,vocab,device):\n",
    "        super(LstmCrf,self).__init__()\n",
    "        #参数\n",
    "        self.input_dim=input_dim\n",
    "        self.hidden_dim=hidden_dim\n",
    "        self.label2id=label2id\n",
    "        self.n_class=len(label2id)\n",
    "        self.vocab=vocab\n",
    "        self.device=device\n",
    "        #嵌入层 input:[batch,seq_len]  output:[batch,seq_len,input_dim]\n",
    "        self.embedding=nn.Embedding(len(vocab),input_dim)\n",
    "        \n",
    "        #LSTM层  input:[batch,seq_len,input_dim] output[batch,seq_len,2*hidden_num]\n",
    "        self.lstm=nn.LSTM(input_size=self.input_dim,hidden_size=self.hidden_dim,num_layers=2,batch_first=True,bidirectional=True)\n",
    "        \n",
    "        #输出映射tag output[batch,n_class]\n",
    "        self.hidden2tag=nn.Linear(2*hidden_dim,self.n_class)\n",
    "        \n",
    "        #转移矩阵，因为是需要学习的，所以用paramet添加,transition[i,j]means transition probability from i to j\n",
    "        self.transition_matrix=nn.Parameter(torch.rand(self.n_class,self.n_class))\n",
    "        self.reset_parameters()\n",
    "    def reset_parameters(self):\n",
    "        init.normal_(self.transition_matrix)\n",
    "        #转移矩阵一行一列都零，目的是为了使从结束转移到其他的概率为0，和从其他转移到开始概率为0，log之后就是非常小，取-10000\n",
    "        #同时通过detach()从计算图中进行分离，使这一部分参数不会改变\n",
    "        #transition_matrix[i,j]表示当前状态为i，转移到状态j的得分，由于只会在句子开头添加<start>，结尾添加<end>所以不存在\n",
    "        #从其他状态转移到<START>，或者<END>转移到其他状态.\n",
    "        self.transition_matrix.detach()[self.label2id['<END>'],:]=-10000\n",
    "        self.transition_matrix.detach()[:,label2id['<START>']]=-10000\n",
    "    def forward(self,input_data,real_length):\n",
    "        \"\"\"\n",
    "        Params:\n",
    "            input_data [batch,seq_len] 从数据迭代器迭代出的一批次样本\n",
    "        Return:\n",
    "            output [batch,seq_len,n_class] LSTM的输出经过线性层生成的发射矩阵\n",
    "        \"\"\"\n",
    "        embed=self.embedding(input_data) #[batch,seq_len,input_dim]\n",
    "        #print(\"embed dim:\",embed.shape)\n",
    "        packed_embed=pack_padded_sequence(input=embed,lengths=real_length,batch_first=True,enforce_sorted = False)#未排序的序列\n",
    "        packed_output,_=self.lstm(packed_embed)\n",
    "        output,_=pad_packed_sequence(packed_output,batch_first=True) #[batch,seq_len,hidden_dim]\n",
    "        #print('\\noutput dim:',output.shape)\n",
    "        output=self.hidden2tag(output)             #[batch,seq,n_class]\n",
    "        return output\n",
    "    def forward_alpha(self,emission,mask):\n",
    "        \"\"\"\n",
    "        主要目的：\n",
    "                计算所有可能路径的分数指数和的对数具体来说就是softmax分母的log值，由于每个时间步的每个输出都有m种可能，\n",
    "                所以必须采用动态规划的方法来计算所有路径的概率 ，具体来说是维护一个n_class维度的向量alpha_t,\n",
    "                使alpha_t[i]代表t时刻，y_t=i的所有路径指数和的对数logZ(y_t=i)，\n",
    "                即alpha_t=[log(Z(y_t=0)),log(Z(y_t=1)),......,log(Z(y_t)=n_class)]\n",
    "                利用关系log(Z(y_t+1=j))=sum(alpha_t+transition[:j]+emission[t+1,j]) 来更新alpha\n",
    "                \n",
    "                具体推导过程参考文章：https://zhuanlan.zhihu.com/p/97829287\n",
    "                代码部分参考pytorch官方tutorial :Advanced: Making Dynamic Decisions and the Bi-LSTM CRF\n",
    "        注意事项：\n",
    "                在每个样本开头添加START最后结束的时候添加END，然后计算log_sum_exp得到最终所有路径的分数的指数和的对数\n",
    "        \n",
    "        Parameter:\n",
    "                emission torch.tensor ---[batch,seq_len,n_class]       LSTM+linear后的output\n",
    "                mask torch.tensor ---[batch,seq_len]                   1=real_word 0=padding_word\n",
    "        Return:\n",
    "                total_score torch.tensor [batch] \n",
    "        \n",
    "        \"\"\"\n",
    "        #mask=torch.tensor(get_mask(lengths),device=device)\n",
    "        emission=emission.to(device)\n",
    "        #print(\"emission.shape:\",emission.shape)\n",
    "        batch_size,seq_len=mask.size()\n",
    "        #time_step=0的时候，状态为START的概率为1，其他为0，对应的log分别为0和-10000，表示起始只有START一条路劲\n",
    "        log_alpha_init=torch.full((batch_size,self.n_class),fill_value=-10000,device=device)\n",
    "        log_alpha_init[:,label2id['<START>']]=0\n",
    "        log_alpha=log_alpha_init #[batch,n_class]\n",
    "        for time_step in range(0,seq_len):\n",
    "            mask_t=mask[:,time_step].unsqueeze(-1) #[batch,1]\n",
    "            emission_t=emission[:,time_step,:].unsqueeze(1).expand(-1,self.n_class,-1) #[batch,n_class,n_class] 进行了复制扩行的操作\n",
    "            \"\"\"\n",
    "            除去batch维度(dim=0)以一个样本进行说明：\n",
    "                这一步的操作是为了进行行的复制，假设一个time_step的发射矩阵为[E_t(0),E_t(1),....,,E_t(n)] [n_class]\n",
    "                是一个torch.tensor([n])的tensor\n",
    "                进行unsqueeze(1)后实际变为 [[E_t(0),E_t(1),....,,E_t(n)]] 注意方括号数 [1,n_class]\n",
    "                进行expand(-self.n_class,-1)的操作后，实际上进行了行的复制\n",
    "                即：\n",
    "                [[E_t(0),E_t(1),....,,E_t(n)],\n",
    "                 [E_t(0),E_t(1),....,,E_t(n)],\n",
    "                 [E_t(0),E_t(1),....,,E_t(n)],\n",
    "                 [E_t(0),E_t(1),....,,E_t(n)],\n",
    "                 [E_t(0),E_t(1),....,,E_t(n)]] 直到行数也为n此时这个矩阵的形状为[n_class,n_class]\n",
    "            \n",
    "            \"\"\"\n",
    "            log_alpha_matrix=log_alpha.unsqueeze(2).expand(-1,-1,self.n_class)         #[batch,n_class,n_class] 进行了复制扩列的操作\n",
    "            \"\"\"\n",
    "            除去batch维度(dim=0)以一个样本进行说明：\n",
    "                log_alpha保存的是当前时刻，每个label对应的所有路径的得分的指数和的对数\n",
    "                这一步的操作是为了进行列的复制，假设一个time_step的log_alpha矩阵为[alpha_t(0),alpha_t(1),....,alpha_t(n)]\n",
    "                是一个torch.tensor([n])的tensor\n",
    "                unsqueeze(2)操作后，实际变为[[alpha_t(0)],[alpha_t(1)],....,[alpha_t(n)]] 注意方括号\n",
    "                expand(-1,-1,self.n_class) 的操作后，实际上进行了列的复制\n",
    "                即\n",
    "                [[alpha_t(0),alpha_t(0),alpha_t(0),alpha_t(0),alpha_t(0),alpha_t(0),alpha_t(0)],\n",
    "                 [alpha_t(1),alpha_t(1),alpha_t(1),alpha_t(1),alpha_t(1),alpha_t(1),alpha_t(1)]\n",
    "                 ,....,\n",
    "                 [alpha_t(n),alpha_t(n),alpha_t(n),alpha_t(n),alpha_t(n),alpha_t(n),alpha_t(n)]]\n",
    "            \n",
    "            \"\"\"\n",
    "            \"\"\"\n",
    "            transition_matrix:\n",
    "                [[T[0,0],T[0,1],....T[0,n]],\n",
    "                 [T[1,0],T[1,1],....T[1,n]],\n",
    "                 [T[2,0],T[2,1],....T[2,n]],\n",
    "                 ...\n",
    "                 [T[n,0],T[n,1],....T[n,n]]\n",
    "            \"\"\"\n",
    "            #print(\"log_alpha_matrix:\",log_alpha_matrix.device,\"\\n\\nlog_M_matrix:\",log_M_matrix.device,\"\\n\\nmask:\",mask_t.device)\n",
    "            add_matrix=log_alpha_matrix+emission_t+self.transition_matrix  #[batch,n_class,n_class]\n",
    "            log_alpha=torch.logsumexp(add_matrix,dim=1)*mask_t+log_alpha*(1-mask_t) #[batch,n_class]\n",
    "            \"\"\"\n",
    "            对上述三个矩阵进行列方向的相加，这里因为加入了batch维度，所以dim=1\n",
    "            满足alpha_t+1(j)=∑(alpha_t(i)+T[i:j]+E[t+1,j] for i in  range(n_class))\n",
    "            \n",
    "            按批次更新log_alpha时，利用到了mask矩阵。当某个数据对应的mask值为1时，表明该个数据还没有穷尽实际长度，\n",
    "            可以更新，于是更新的数值乘以1保留，原来的log_alpha值乘以1-mask=0，消除。如果当前mask值为0，则表示其实\n",
    "            这个数据已经结束，此时虽然新的log_alpha_matrix在对应的位置为0，但是依然存在转移矩阵的得分，所以\n",
    "            要乘以0取消更新，最后将两者相加，最终更新该数据的log_alpha。\n",
    "            \"\"\"\n",
    "        alpha=log_alpha+self.transition_matrix[:,label2id['<END>']].unsqueeze(0)#最后，加上到<END>标签的转移分数，得到最终结果。\n",
    "        total_score=torch.logsumexp(alpha,dim=1) #[batch]\n",
    "        return total_score\n",
    "    def sentence_score(self,emission,labels,mask):\n",
    "        \"\"\"\n",
    "        主要目的：\n",
    "            计算一个批次的标签序列得分\n",
    "        Parameters:\n",
    "            emission 发射矩阵torch.tensor [batch,seq_len,n_class]\n",
    "            labels   标签矩阵 torch.tensor [batch,seq_len]\n",
    "            mask     掩码矩阵 torch.tensor [batch,seq_len]\n",
    "        Return:\n",
    "            socores  得分矩阵 torch.tensor [batch_size,1]\n",
    "        \"\"\"\n",
    "        #mask=torch.tensor(get_mask(lengths),device=device)\n",
    "        batch_size,seq_len,n_class=emission.size()\n",
    "        #[batch,seq_len+1]\n",
    "        labels=torch.cat([labels.new_full((batch_size,1),fill_value=label2id['<START>']),labels],1) #用new_full保持和labels同样的dtype和device\n",
    "        scores=emission.new_zeros(batch_size) \n",
    "        for t in range(seq_len):\n",
    "            mask_t=mask[:,t] #[batch,1]\n",
    "            emission_t=emission[:,t,:] #[batch,n_class]\n",
    "            #一个读取一个样本的发射分数，和对应的标签，计算在当前时间步的发射得分\n",
    "            #最后所有Batch并在dim=0的维度连接起来最后[batch,1]；t+1是因为加了一个<START>\n",
    "            emit_score=torch.cat([each_score[next_label].unsqueeze(-1) for each_score,next_label in zip(emission_t,labels[:,t+1])],dim=0)\n",
    "            transition_score=torch.stack([self.transition_matrix[labels[b,t],labels[b,t+1]]for b in range(batch_size)])#[batch,1]\n",
    "            #transtion_score是计算前一个时间步到当前时间步的标签转移得分所以从t开始\n",
    "            scores+=(emit_score+transition_score)*mask_t #如果当前时间步的单词是填充的话就不计算分数\n",
    "        \"\"\"\n",
    "        添加最后一步到<END>的转移分数，由于每个步骤的实际长度不同，用mask[b:].sum()-1+1来求最后一个单词数的位置，实际上是\n",
    "        \"\"\"\n",
    "        transition_to_end=torch.stack([self.transition_matrix[label[mask[b,:].sum().long()],label2id['<END>']]for b,label in enumerate(labels)])\n",
    "        scores+=transition_to_end\n",
    "        return scores \n",
    "        \n",
    "    def neg_log_likelihood(self,emission,labels,mask):\n",
    "        mask=mask.cuda()\n",
    "        alpha_score=self.forward_alpha(emission,mask)\n",
    "        score_sentence=self.sentence_score(emission,labels,mask)\n",
    "        loss=(alpha_score-score_sentence).sum()\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viterbi_decode( feats, mask,transition,tag2idx):\n",
    "    \"\"\"\n",
    "    :param feats: (seq_len, batch_size, tag_size)\n",
    "    :param mask: (seq_len, batch_size)\n",
    "    :return best_path: (seq_len, batch_size)\n",
    "    \"\"\"\n",
    "    feats=feats.transpose(1,0).contiguous()\n",
    "    mask=mask.transpose(1,0).contiguous()\n",
    "    seq_len, batch_size, tag_size = feats.size()\n",
    "    # initialize scores in log space\n",
    "    scores = feats.new_full((batch_size, tag_size), fill_value=-10000)\n",
    "    scores[:, tag2idx['<START>']] = 0\n",
    "    pointers = []\n",
    "    # forward\n",
    "    for t, feat in enumerate(feats):\n",
    "        # broadcast dimension: (batch_size, next_tag, current_tag)\n",
    "        scores_t = scores.unsqueeze(1) + transition.unsqueeze(0)  # (batch_size, tag_size, tag_size)\n",
    "        # max along current_tag to obtain: next_tag score, current_tag pointer\n",
    "        scores_t, pointer = torch.max(scores_t, -1)  # (batch_size, tag_size), (batch_size, tag_size)\n",
    "        scores_t += feat\n",
    "        pointers.append(pointer)\n",
    "        mask_t = mask[t].unsqueeze(-1)  # (batch_size, 1)\n",
    "        scores = scores_t * mask_t + scores * (1 - mask_t)\n",
    "    pointers = torch.stack(pointers, 0) # (seq_len, batch_size, tag_size)\n",
    "    scores +=transition[:,tag2idx['<END>']].unsqueeze(0)\n",
    "    best_score, best_tag = torch.max(scores, -1)  # (batch_size, ), (batch_size, )\n",
    "    # backtracking\n",
    "    best_path = best_tag.unsqueeze(-1).tolist() # list shape (batch_size, 1)\n",
    "    for i in range(batch_size):\n",
    "        best_tag_i = best_tag[i]\n",
    "        seq_len_i = int(mask[:, i].sum())\n",
    "        for ptr_t in reversed(pointers[:seq_len_i, i]):\n",
    "        # ptr_t shape (tag_size, )\n",
    "            best_tag_i = ptr_t[best_tag_i].item()\n",
    "            best_path[i].append(best_tag_i)\n",
    "        # pop first tag\n",
    "        best_path[i].pop()\n",
    "        # reverse order\n",
    "        best_path[i].reverse()\n",
    "        \n",
    "    return np.array(best_path).T\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "维特比预测最佳路劲这一步其实跟模型没有太大关系，因为本身就要禁止梯度的传递。并且要不断修改这个函数，如果放在类里面的话每次都要重新加载模型，加词向量比较麻烦，所以就单独拿出来在外面好修改"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.训练及评价模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 加载glove词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "有3442个 OOV单词.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
       "        [-0.0828,  0.6720, -0.1499,  ..., -0.1918, -0.3785, -0.0659],\n",
       "        ...,\n",
       "        [ 0.3803,  0.0872, -0.1009,  ..., -0.3404, -0.2451, -0.0514],\n",
       "        [-0.9065,  0.0039,  0.0859,  ..., -0.9103,  0.2001, -0.1049],\n",
       "        [ 0.4403, -0.4592,  0.1195,  ...,  0.5576,  0.1360, -0.1152]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device=torch.device('cuda' if torch.cuda.is_available else  'cpu')\n",
    "model=LstmCrf(300,100,label2id,vocab.stoi,device)\n",
    "glove_vocab=Vocab.GloVe(name='840B',dim=300,cache=r'E:\\NLP_jupyternotebook\\Fudan_NLP_beginner\\.vector_cache')\n",
    "\"\"\"\n",
    "将词向量作为每个词的特征向量，先创建一个矩阵，在通过copy_()函数赋值给模型中的self.embedding\n",
    "\"\"\"\n",
    "def load_pretrain_embedding(words,pretrained_vocab): #两个都是vocab类\n",
    "    embed=torch.zeros(len(words),pretrained_vocab.vectors[0].shape[0])\n",
    "    oov_count=0\n",
    "    for i,word in enumerate(words):#词典中的词可能没有对应的词向量，所以用try except来避免异常导致的程序中断\n",
    "        try:\n",
    "            idx=pretrained_vocab.stoi[word]\n",
    "            embed[i,:]=pretrained_vocab.vectors[idx]\n",
    "        except KeyError:\n",
    "            oov_count+=1\n",
    "    if oov_count>0:\n",
    "        print(\"有%d个 OOV单词.\"%oov_count)\n",
    "    return embed\n",
    "model.embedding.weight.data.copy_(load_pretrain_embedding(vocab.itos,glove_vocab))\n",
    "#model.embedding.weight.requires_grad=False #已经是预训练好的词向量，这里就不再需要进行更新"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "试过了选择6B 100d的词向量，发现OOV单词过多了，所以就改成了840B,300d的词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 6.5786e-02,  1.0397e-01, -2.5876e-01, -1.1300e+00, -1.7075e-01,\n",
       "         6.8380e-02,  1.5320e+00,  3.6261e-01, -5.9450e-01, -1.7887e-01,\n",
       "        -1.0000e+04], grad_fn=<SelectBackward>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.transition_matrix[:,label2id['<END>']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter=DataLoader(dataset=train_set,batch_size=32,collate_fn=collate_fn)\n",
    "test_iter=DataLoader(dataset=test_set,batch_size=16,collate_fn=collate_fn)\n",
    "lr,epoch=0.01,5\n",
    "optimizer=torch.optim.Adam(model.parameters(),lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "def Assessment(test_iter,net,device,label2id):\n",
    "    \"\"\"\n",
    "    主要目的：\n",
    "            构建混淆矩阵confmatrix并计算F1,RECALL,PRECISION\n",
    "            confmatrix(i,j)表示实际类i被预测成j的样本数，当且仅当i=j，即矩阵主对角线上的元素是被正确预测的\n",
    "            总体准确率:all_prediction=sum(diag(confmatrix))/sum(confmatrix)\n",
    "            某个类别的精度 label_i_precision=confmatrix(i,i)/sum(confmatrix(j,i)for j in range(n_class))\n",
    "            某个类别召回率 label_i_recall=confmatrix(i,i)/sum(confmatrix(i,j)for j in range(n_class))\n",
    "            宏召回率 all_recall=avg(label_i_prediciton for i in range(n_class))\n",
    "            宏精度 all_precision=avg(label_i_precision for i in range(n_class))\n",
    "            某个类别的F1 F1=2*precision*recall/(precision+recall)\n",
    "    \n",
    "    \"\"\"\n",
    "    n_class=len(label2id)\n",
    "    #创建混淆矩阵\n",
    "    confmatrix=np.zeros((n_class,n_class),dtype=np.int32)\n",
    "    net.eval()\n",
    "    ci=0\n",
    "    with torch.no_grad():\n",
    "        for X,y,lengths in test_iter:\n",
    "            ci+=1\n",
    "            X=X.long()\n",
    "            y=y.long()\n",
    "            X=X.to(device)\n",
    "            y=y.to(device)\n",
    "            mask=get_mask(lengths)\n",
    "            mask=mask.cuda()\n",
    "            emission=net.forward(X,lengths)\n",
    "            #print(emission.device)\n",
    "            y_hat=viterbi_decode(emission,mask,model.transition_matrix,label2id)\n",
    "            #print('mask',mask)\n",
    "            #print('y',y[0])\n",
    "            #if(ci%10==0):\n",
    "                #print('实际的标签序列：',[id2label[x] for x in y[0]],\"实际标签长度\",len(y[0]))\n",
    "                #print('预测的最好标签序列：',[id2label[x] for x in y_hat[0]],\"预测标签长度\",len(y_hat[0]))\n",
    "            for b in range(len(y_hat)):\n",
    "                for i in range(int(sum(mask[b]))):\n",
    "                    true_label_idx=y[b][i]\n",
    "                    predict_label_idx=y_hat[b][i]\n",
    "                    #predict_label_idx=y[b][i]\n",
    "                    confmatrix[true_label_idx][predict_label_idx]+=1\n",
    "    net.train()\n",
    "    #求整体准确率\n",
    "    total_sum=confmatrix.sum()\n",
    "    correct_sum=(np.diag(confmatrix)).sum()\n",
    "    total_accuracy=100*float(correct_sum)/float(total_sum)\n",
    "    def calculate_precision(confMatrix,labelidx):\n",
    "        \"\"\"求某一个类别的precision\"\"\"\n",
    "        label_total_sum = confMatrix.sum(axis=0)[labelidx]\n",
    "        label_correct_sum = confMatrix[labelidx][labelidx]\n",
    "        precision=0\n",
    "        if label_total_sum != 0:\n",
    "            precision = 100*float(label_correct_sum)/float(label_total_sum)\n",
    "        return precision\n",
    "    \n",
    "    def calculate_recall(confMatrix,labelidx):\n",
    "        label_total_sum = confMatrix.sum(axis=1)[labelidx]\n",
    "        label_correct_sum = confMatrix[labelidx][labelidx]\n",
    "        recall = 0\n",
    "        if label_total_sum != 0:\n",
    "            recall = 100*float(label_correct_sum)/float(label_total_sum)\n",
    "        return recall\n",
    "    \n",
    "    def calculate_f1(prediction,recall):\n",
    "        if (prediction+recall==0):\n",
    "            return 0\n",
    "        return round(2*prediction*recall/(prediction+recall),2)\n",
    "    precisions=dict([labelname,calculate_precision(confmatrix,labelidx)] for labelname,labelidx in label2id.items())\n",
    "    recalls=dict([labelname,calculate_recall(confmatrix,labelidx)] for labelname,labelidx in label2id.items())\n",
    "    F1s=dict([labelname,calculate_f1(precisions[labelname],recalls[labelname])] for labelname,_ in label2id.items())\n",
    "    return confmatrix,total_accuracy,precisions,recalls,F1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "def train(train_iter,test_iter,net,optimizer,device,num_epochs):\n",
    "    net=net.to(device)\n",
    "    print(\"training on \",device)\n",
    "    batch_count=0\n",
    "    for epoch in range(num_epochs):\n",
    "        net.train()\n",
    "        train_l_sum,train_acc_sum,n,start=0.0,0.0,0,time.time()\n",
    "        for X,y,lengths in tqdm(train_iter):\n",
    "            X=X.long()\n",
    "            y=y.long()\n",
    "            X=X.to(device)\n",
    "            y=y.to(device)\n",
    "            n+=y.shape[0]\n",
    "            mask=get_mask(lengths)\n",
    "            emission=net.forward(X,lengths)\n",
    "            #print(emission.shape)\n",
    "            #return 0\n",
    "            l=net.neg_log_likelihood(emission,y,mask)\n",
    "            optimizer.zero_grad()\n",
    "            l.backward()\n",
    "            #torch.nn.utils.clip_grad_norm_(parameters=net.parameters(), max_norm=10)\n",
    "            optimizer.step()\n",
    "            train_l_sum+=l\n",
    "            #print([mask.sum(dim=1)]==[len(x) for x in y_hat])\n",
    "            #confmatrix,total_accuracy,precisions,recalls,F1s=Assessment(test_iter,net,device,label2id)\n",
    "            #print(\"epoch %d\\tloss %.3f\\ttotal_accuracy %.2f\\n各标签查准率:\"%(epoch,train_l_sum,total_accuracy),precisions,\"\\n各标签召回率:\",recalls,\"\\n各标签的F1值:\",F1s,'\\n',confmatrix)\n",
    "        train_l_avg=train_l_sum/n\n",
    "        confmatrix,total_accuracy,precisions,recalls,F1s=Assessment(test_iter,net,device,label2id)\n",
    "        print(\"epoch %d\\tloss %.3f\\ttotal_accuracy %.2f\\n各标签查准率:\"%(epoch,train_l_avg,total_accuracy),precisions,\"\\n各标签召回率:\",recalls,\"\\n各标签的F1值:\",F1s,'\\n',confmatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|                                                                                          | 0/439 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training on  cuda\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 439/439 [09:02<00:00,  1.24s/it]\n",
      "  0%|                                                                                          | 0/439 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0\tloss 1.506\ttotal_accuracy 93.77\n",
      "各标签查准率: {'<START>': 0, 'I-LOC': 31.65680473372781, 'O': 98.44238382996457, 'B-LOC': 83.94833948339483, 'B-PER': 84.29423459244533, 'B-ORG': 73.76861397479954, 'I-PER': 83.61204013377926, 'I-ORG': 50.85995085995086, 'B-MISC': 64.72148541114058, 'I-MISC': 29.315068493150687, '<END>': 0} \n",
      "各标签召回率: {'<START>': 0, 'I-LOC': 83.26848249027238, 'O': 98.71235667458929, 'B-LOC': 82.27848101265823, 'B-PER': 79.1044776119403, 'B-ORG': 77.54364840457556, 'I-PER': 65.38796861377507, 'I-ORG': 24.790419161676645, 'B-MISC': 69.51566951566952, 'I-MISC': 49.53703703703704, '<END>': 0} \n",
      "各标签的F1值: {'<START>': 0, 'I-LOC': 45.87, 'O': 98.58, 'B-LOC': 83.11, 'B-PER': 81.62, 'B-ORG': 75.61, 'I-PER': 73.39, 'I-ORG': 33.33, 'B-MISC': 67.03, 'I-MISC': 36.83, '<END>': 0} \n",
      " [[    0     0     0     0     0     0     0     0     0     0     0]\n",
      " [    0   214    14    20     1     0     0     6     0     2     0]\n",
      " [    0    29 37794    30    36    62    12   129   110    85     0]\n",
      " [    0    12    22  1365    19   206     0     6    28     1     0]\n",
      " [    0     3    78    18  1272    98   133     4     1     1     0]\n",
      " [    0    10    79   117    39  1288     0    43    76     9     0]\n",
      " [    0    61   194     8   127     0   750     5     1     1     0]\n",
      " [    0   330    96    48     1    36     1   207     5   111     0]\n",
      " [    0     2    74    19    13    54     1     3   488    48     0]\n",
      " [    0    15    41     1     1     2     0     4    45   107     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0     0]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 439/439 [09:07<00:00,  1.25s/it]\n",
      "  0%|                                                                                          | 0/439 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1\tloss 0.390\ttotal_accuracy 94.31\n",
      "各标签查准率: {'<START>': 0, 'I-LOC': 42.295081967213115, 'O': 97.9980367844596, 'B-LOC': 78.67607162235485, 'B-PER': 85.59670781893004, 'B-ORG': 77.00879765395895, 'I-PER': 85.65697091273822, 'I-ORG': 68.4375, 'B-MISC': 67.34693877551021, 'I-MISC': 30.303030303030305, '<END>': 0} \n",
      "各标签召回率: {'<START>': 0, 'I-LOC': 50.19455252918288, 'O': 99.08585159453601, 'B-LOC': 87.40204942736588, 'B-PER': 77.61194029850746, 'B-ORG': 79.04876580373269, 'I-PER': 74.45510026155188, 'I-ORG': 26.227544910179642, 'B-MISC': 70.51282051282051, 'I-MISC': 41.666666666666664, '<END>': 0} \n",
      "各标签的F1值: {'<START>': 0, 'I-LOC': 45.91, 'O': 98.54, 'B-LOC': 82.81, 'B-PER': 81.41, 'B-ORG': 78.02, 'I-PER': 79.66, 'I-ORG': 37.92, 'B-MISC': 68.89, 'I-MISC': 35.09, '<END>': 0} \n",
      " [[    0     0     0     0     0     0     0     0     0     0     0]\n",
      " [    0   129    31    90     0     1     1     2     2     1     0]\n",
      " [    0     8 37937    42    37    53    13    40    82    75     0]\n",
      " [    0     4    37  1450    14   114     2     3    34     1     0]\n",
      " [    0     1   114    27  1248    94   113     1    10     0     0]\n",
      " [    0     2   133    96    24  1313     0    50    43     0     0]\n",
      " [    0    20   139    10   117     3   854     2     1     1     0]\n",
      " [    0   135   158   103     2    94    13   219    13    98     0]\n",
      " [    0     0   104    21    16    32     0     3   495    31     0]\n",
      " [    0     6    59     4     0     1     1     0    55    90     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0     0]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 439/439 [09:13<00:00,  1.26s/it]\n",
      "  0%|                                                                                          | 0/439 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2\tloss 0.154\ttotal_accuracy 95.18\n",
      "各标签查准率: {'<START>': 0.0, 'I-LOC': 60.182370820668694, 'O': 98.38277735888302, 'B-LOC': 88.29787234042553, 'B-PER': 91.19031607262946, 'B-ORG': 81.38181818181818, 'I-PER': 92.01793721973094, 'I-ORG': 51.94805194805195, 'B-MISC': 74.54545454545455, 'I-MISC': 51.492537313432834, '<END>': 0} \n",
      "各标签召回率: {'<START>': 0, 'I-LOC': 77.04280155642023, 'O': 98.82989004100608, 'B-LOC': 85.05123568414707, 'B-PER': 84.32835820895522, 'B-ORG': 67.36905478627332, 'I-PER': 89.45074106364429, 'I-ORG': 67.06586826347305, 'B-MISC': 70.08547008547009, 'I-MISC': 63.888888888888886, '<END>': 0} \n",
      "各标签的F1值: {'<START>': 0, 'I-LOC': 67.58, 'O': 98.61, 'B-LOC': 86.64, 'B-PER': 87.63, 'B-ORG': 73.72, 'I-PER': 90.72, 'I-ORG': 58.55, 'B-MISC': 72.25, 'I-MISC': 57.02, '<END>': 0} \n",
      " [[    0     0     0     0     0     0     0     0     0     0     0]\n",
      " [    0   198    20    19     0     0     2    14     0     4     0]\n",
      " [    1    20 37839    30    43    48    10   178    66    52     0]\n",
      " [    0     5    47  1411    23   118     1    21    31     2     0]\n",
      " [    0     1   105    31  1356    40    68     5     2     0     0]\n",
      " [    0     3   118    68    22  1119     1   287    41     2     0]\n",
      " [    0     5    86     4    20     1  1026     4     0     1     0]\n",
      " [    0    91    83    20     5    23     6   560    12    35     0]\n",
      " [    0     1   112    13    18    26     0     6   492    34     0]\n",
      " [    0     5    51     2     0     0     1     3    16   138     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0     0]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 439/439 [09:14<00:00,  1.26s/it]\n",
      "  0%|                                                                                          | 0/439 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3\tloss 0.073\ttotal_accuracy 95.55\n",
      "各标签查准率: {'<START>': 0, 'I-LOC': 69.6969696969697, 'O': 98.59448747033821, 'B-LOC': 84.6685878962536, 'B-PER': 89.63058976020739, 'B-ORG': 83.68314150304671, 'I-PER': 94.98580889309366, 'I-ORG': 61.469933184855236, 'B-MISC': 73.11827956989248, 'I-MISC': 44.082840236686394, '<END>': 0} \n",
      "各标签召回率: {'<START>': 0, 'I-LOC': 62.64591439688716, 'O': 98.75414631598193, 'B-LOC': 88.5473176612417, 'B-PER': 86.00746268656717, 'B-ORG': 74.41300421432872, 'I-PER': 87.53269398430689, 'I-ORG': 66.10778443113773, 'B-MISC': 77.4928774928775, 'I-MISC': 68.98148148148148, '<END>': 0} \n",
      "各标签的F1值: {'<START>': 0, 'I-LOC': 65.98, 'O': 98.67, 'B-LOC': 86.56, 'B-PER': 87.78, 'B-ORG': 78.78, 'I-PER': 91.11, 'I-ORG': 63.7, 'B-MISC': 75.24, 'I-MISC': 53.79, '<END>': 0} \n",
      " [[    0     0     0     0     0     0     0     0     0     0     0]\n",
      " [    0   161    28    32     1     1     1    13     0    20     0]\n",
      " [    0    10 37810    35    53    47    18   135    75   104     0]\n",
      " [    0     3    36  1469    11    92     1    15    29     3     0]\n",
      " [    0     0   103    30  1383    49    25     2    16     0     0]\n",
      " [    0     0   105    80    26  1236     0   166    44     4     0]\n",
      " [    0     3    57    20    46     3  1004     7     3     4     0]\n",
      " [    0    50    99    46     4    21     7   552    21    35     0]\n",
      " [    0     0    70    18    19    28     0     4   544    19     0]\n",
      " [    0     4    41     5     0     0     1     4    12   149     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0     0]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 439/439 [09:03<00:00,  1.24s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4\tloss 0.043\ttotal_accuracy 95.82\n",
      "各标签查准率: {'<START>': 0, 'I-LOC': 67.76556776556777, 'O': 98.50563958625708, 'B-LOC': 86.5351131746953, 'B-PER': 90.59438275636839, 'B-ORG': 84.49511400651465, 'I-PER': 92.26569608735214, 'I-ORG': 70.83333333333333, 'B-MISC': 74.56395348837209, 'I-MISC': 41.23076923076923, '<END>': 0} \n",
      "各标签召回率: {'<START>': 0, 'I-LOC': 71.98443579766537, 'O': 98.99704860657664, 'B-LOC': 89.87341772151899, 'B-PER': 86.25621890547264, 'B-ORG': 78.08549066827213, 'I-PER': 88.40453356582388, 'I-ORG': 61.07784431137725, 'B-MISC': 73.07692307692308, 'I-MISC': 62.03703703703704, '<END>': 0} \n",
      "各标签的F1值: {'<START>': 0, 'I-LOC': 69.81, 'O': 98.75, 'B-LOC': 88.17, 'B-PER': 88.37, 'B-ORG': 81.16, 'I-PER': 90.29, 'I-ORG': 65.59, 'B-MISC': 73.81, 'I-MISC': 49.54, '<END>': 0} \n",
      " [[    0     0     0     0     0     0     0     0     0     0     0]\n",
      " [    0   185    18    35     0     1     3     6     1     8     0]\n",
      " [    0    10 37903    31    47    57    16    79    63    81     0]\n",
      " [    0     0    39  1491    18    67     2     6    32     4     0]\n",
      " [    0     0    76    24  1387    54    56     3     8     0     0]\n",
      " [    0     1   110    84    31  1297     0    97    37     4     0]\n",
      " [    0     0    81     6    31     0  1014     9     1     5     0]\n",
      " [    0    68   124    33     3    25     7   510    13    52     0]\n",
      " [    0     1    83    16    13    34     0     5   513    37     0]\n",
      " [    0     8    44     3     1     0     1     5    20   134     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0     0]]\n"
     ]
    }
   ],
   "source": [
    "train(train_iter,test_iter,model,optimizer,device,epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'B-ORG': 6321,\n",
       "         'O': 169578,\n",
       "         'B-MISC': 3438,\n",
       "         'B-PER': 6600,\n",
       "         'I-PER': 4528,\n",
       "         'B-LOC': 7140,\n",
       "         'I-ORG': 3704,\n",
       "         'I-MISC': 1155,\n",
       "         'I-LOC': 1157})"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_nums"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练集的loss在不断降低，同时测试集的准确率也在不断的提高，各个标签的F1值出现了波动，但总体是上升趋势。观察发现各个标签的F1值与标签的数量成正相关"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}