speech_recognition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Speech recognition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use a simple [database](https://www.dropbox.com/s/c12fmsctfwwov5d/sounds.zip) composed of 12 french words pronounced about 25 times by different speakers. You need to download the dataset, unzip and replace the directory \"sounds\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'zidane', 'stade', 'beckham', 'sofoot', 'biere', 'girondins', 'manette', 'chaussure', 'jeuvideo', 'gants', 'cocacola', 'ballon'}\n"
     ]
    }
   ],
   "source": [
    "with open(\"sounds/wavToTag.txt\") as f:\n",
    "    labels = array([l.replace('\\n', '') for l in f.readlines()])\n",
    "#print(labels) # 245 words\n",
    "print(set(labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precompute all MFCCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time used: 93.340004s.\n"
     ]
    }
   ],
   "source": [
    "import librosa\n",
    "import time\n",
    "\n",
    "mfccs = {}\n",
    "start = time.clock()\n",
    "for i in range(len(labels)):\n",
    "    y, sr = librosa.load('sounds/{}.wav'.format(i))\n",
    "    mfcc = librosa.feature.mfcc(y, sr, n_mfcc=13)\n",
    "    mfccs[i] = mfcc.T\n",
    "print(\"Time used: %fs.\"%(time.clock() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40, 13)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mfccs[0].shape # mfccs is a dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Leave P Out Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0,   6,   8,  38,  60,  61,  64,  67,  72,  76,  81,  86,  89,\n",
       "        95, 100, 104, 106, 111, 122, 134, 186, 187, 201, 202, 212, 226, 228])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find(labels == 'chaussure')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 2, 3, 4, 5, 6]\n",
      "[4 5 6 2 3 1]\n"
     ]
    }
   ],
   "source": [
    "train = [1, 2, 3]\n",
    "train += [4, 5, 6]\n",
    "print(train)\n",
    "array_train = array(train)\n",
    "shuffle(array_train)\n",
    "print(array_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generate_train_test_set(P):\n",
    "    train = []\n",
    "    test = []\n",
    "\n",
    "    for s in set(labels):\n",
    "        all = find(labels == s)\n",
    "        shuffle(all)\n",
    "        train += all[:-P].tolist()\n",
    "        test += all[-P:].tolist()\n",
    "        \n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1., -1., -1., ..., -1., -1., -1.],\n",
       "       [-1., -1., -1., ..., -1., -1., -1.],\n",
       "       [-1., -1., -1., ..., -1., -1., -1.],\n",
       "       ..., \n",
       "       [-1., -1., -1., ..., -1., -1., -1.],\n",
       "       [-1., -1., -1., ..., -1., -1., -1.],\n",
       "       [-1., -1., -1., ..., -1., -1., -1.]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ones((len(labels), len(labels))) * -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from dtw import dtw\n",
    "\n",
    "# We use DP to speed up multiple tests\n",
    "D = ones((len(labels), len(labels))) * -1\n",
    "\n",
    "def cross_validation(train, test):\n",
    "    score = 0.0\n",
    "    for i in test:\n",
    "        x = mfccs[i]\n",
    "        dmin, jmin = inf, -1\n",
    "        for j in train:\n",
    "            y = mfccs[j]\n",
    "            d = D[i, j]\n",
    "            \n",
    "            if d == -1:\n",
    "                d, _, _, _ = dtw(x, y, dist=lambda x, y: norm(x - y, ord=1))\n",
    "                D[i, j] = d                \n",
    "            if d < dmin:\n",
    "                dmin = d\n",
    "                jmin = j\n",
    "        score += 1.0 if (labels[i] == labels[jmin]) else 0.0\n",
    "        \n",
    "    return score / len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recognition rate 75.0%\n",
      "Time used: 101.742457s\n"
     ]
    }
   ],
   "source": [
    "start = time.clock()\n",
    "train, test = generate_train_test_set(P=1)\n",
    "rec_rate = cross_validation(train, test)\n",
    "print('Recognition rate {}%'.format(100. * rec_rate))\n",
    "print(\"Time used: %fs\"%(time.clock() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next plot may take a while to compute!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "start = time.clock()\n",
    "P = arange(1, 10)\n",
    "N = 5\n",
    "\n",
    "rec = []\n",
    "for p in P:\n",
    "    r = [cross_validation(*generate_train_test_set(p)) for _ in range(N)]\n",
    "    rec.append(r)\n",
    "    \n",
    "rec = array(rec)\n",
    "rec = rec.reshape((N, -1))\n",
    "\n",
    "errorbar(P - 0.5, mean(rec, axis=0), yerr=std(rec, axis=0))\n",
    "xticks(P - 0.5, P)\n",
    "ylim(0, 1)\n",
    "print(\"Time used: %fs\"%(time.clock() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}