thesis.tex

\documentclass[12pt]{report}
\usepackage{suthesis}

% -- Imports --
% (general libraries)
\usepackage{times,latexsym,amsfonts,amssymb,amsmath,graphicx,bbm,rotating}
\usepackage{enumitem,multirow,hhline,stmaryrd,bussproofs,mathtools,siunitx,arydshln}
\usepackage{booktabs,xcolor,csquotes}
% (custom libraries)
\usepackage{fitch}
% (inline references)
\usepackage{natbib}
% (tikz)
\usepackage{tikz}
\usepackage{tikz-dependency,pifont}
\usetikzlibrary{shapes.arrows,chains,positioning,automata,trees,calc}
\usetikzlibrary{patterns}
\usetikzlibrary{decorations.pathmorphing,decorations.markings}
% (print algorithms)
\usepackage[ruled,lined,linesnumbered]{algorithm2e}
\makeatletter 
\g@addto@macro{\@algocf@init}{\SetKwInOut{Parameter}{Parameters}} 
\makeatother

%% (custom)
\input std-macros.tex
\input macros.tex	

% (paper compilation hacks)
\def\newcite#1{\citet{#1}}
\def\cite#1{\citep{#1}}

\usepackage{epigraph}
\renewcommand{\epigraphsize}{\normalsize}
\setlength{\epigraphwidth}{0.8\textwidth}
\usepackage{pstricks} % for overlay latex on top of images
\usepackage{pgfplots}

\newcommand{\edit}[1]{{#1}} %\color{blue} 

% latex
\usepackage{hyperref}
\usepackage[hyphenbreaks]{breakurl}

\definecolor{lightred}{HTML}{BB4C4C}
\definecolor{lightgreen}{HTML}{44B178}
\definecolor{lightblue}{HTML}{3B8CDA}
\definecolor{lightpurple}{HTML}{C183E8}
\newcommand{\error}[1]{{\color{red} #1}}
\newcommand{\ngram}{$n$-gram}
\newcommand{\word}[1]{``#1''}
\newcommand{\bfit}[1]{\textbf{\textit{#1}}}
\newcommand{\unk}{$<$\texttt{unk}$>$}
\newcommand{\sos}{$<$\texttt{s}$>$}
\newcommand{\eos}{$<$\texttt{/s}$>$}
\newcommand{\eq}[1]{Eq.~(\ref{#1})}
\newcommand{\figref}[1]{Figure~\ref{#1}}
\newcommand{\nlmtext}{neural language models} % probabilistic 
\newcommand{\nlm}{NLM}
\newcommand{\nlms}{NLMs}
\newcommand{\secref}[1]{Section~\ref{#1}}
\newcommand{\algo}[1]{Algorithm~\ref{#1}}
\newcommand{\biformat}[1]{\textbf{\textit{#1}}}

% math operators
\DeclareMathOperator{\trace}{Tr}
\DeclareMathOperator{\sigm}{sigm}
\DeclareMathOperator{\sigmoid}{sigmoid}
\DeclareMathOperator{\softmax}{softmax}
\DeclareMathOperator{\Index}{Index}

% math
\newcommand{\MB}[1]{\mbox{\boldmath{$#1$}}} % boldface for matrix and vectors
% derivative fractions
\newcommand{\parfrac}[1]{\frac{\partial}{\partial #1}} 
% fraction derivatives
\newcommand{\fracder}[2]{\frac{\partial #1}{\partial #2}}
% sum and product
\newcommand{\sumkn}{\sum_{k=1}^{n}} %I use this when deriving backprop eqns. n is the LSTM block size and k is a better choice than i because it won't get confused with the input gate
% plus equal
\newcommand{\pluseq}{\mathrel{+}=}
% transpose
\newcommand{\tp}[1]{#1^\top}
% round bracket
\newcommand{\open}[1]{\left(#1\right)}
\newcommand{\paren}[1]{\left(#1\right)}
% norm
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
% grad
\newcommand{\grad}[1]{\nabla #1}

%%% math commands %%%
% variables 
\usepackage{bm}
\newcommand{\x}[1]{\bm{x}_{#1}}
\newcommand{\y}[1]{\bm{y}_{#1}}
\newcommand{\z}[1]{\bm{z}_{#1}}
\newcommand{\s}[1]{\bm{s}_{#1}}
\newcommand{\prob}[1]{\bm{p}_{#1}}
\newcommand{\der}[1]{d\bm{#1}}
\newcommand{\W}[1]{\bm{W}_{#1}}
\newcommand{\inR}[1]{\in \mathbb{R}^{#1}}
\newcommand{\yrange}[2]{y_{\overline{#1,#2}}}
\newcommand{\ytop}[1]{y^{(#1)}}
\newcommand{\thetav}{\bm{\theta}}

\newcommand{\edot}{\circ}
% lstm
\newcommand{\mem}[1]{\bm{c}_{#1}}
\newcommand{\ig}{\bm{i}_{t}}
\newcommand{\fg}{\bm{f}_{t}}
\newcommand{\og}{\bm{o}_{t}}
\newcommand{\hg}{\MB{\hat{h}_t}}
\newcommand{\hparam}{\MB{T}_{\text{h}}} % hidden params 
\newcommand{\xparam}{\MB{T}_{\text{x}}} % hidden params 
\newcommand{\ut}{\bm{u}_t}

% NMT
\newcommand{\tgt}[1]{y_{#1}} % target word
\newcommand{\src}[1]{x_{#1}} % source word
\newcommand{\al}{\MB{a}_{t}} % align
\newcommand{\hid}[1]{\bm{h}_{#1}}
\newcommand{\his}{\MB{\bar{h}}_{s}} % hidden top source
\newcommand{\hs}{\MB{\tilde{h}}_{t}} %h softmax
\newcommand{\hd}[1]{\MB{h}_{#1}} % hidden target 
\newcommand{\hb}[1]{\MB{\bar{h}}_{#1}} % hidden source
\newcommand{\rnn}{\MB{T}_{\text{rnn}}} % RNN
\newcommand{\lstm}{\MB{T}_{\text{lstm}}} % LSTM 
\newcommand{\hi}{\MB{h}_{t}} % hidden top
\newcommand{\co}{\MB{c}_{t}} % context


%% theorem environments
\usepackage{amsthm} %for the proof environment
\newtheorem{remark}{{\it Remark}}
\newtheorem{lemma}{{\it Lemma}}
\newtheorem{corollary}{{\it Corollary}}
\newcommand{\lemmaref}[1]{Lemma~\ref{#1}}
\newcommand{\remarkref}[1]{Remark~\ref{#1}}

%% Chapter 3
\newcommand{\unkword}[1]{{\bf {\it {\underline{#1}}}}}
\newcommand{\unksym}{{\it \texttt{\underline{unk}}}}
\newcommand{\eossym}{$<$\texttt{eos}$>$}
\newcommand{\unkcopy}[1]{{\bf {\it {\texttt{\underline{unk}$_{#1}$ }}}}}
\newcommand{\unknull}{{\bf {\it {\texttt{\underline{unk}$_{\emptyset}$}}}}}
\newcommand{\unktext}[1]{{\bf {\it \texttt{\underline{unk}}$_{#1}$}}}
\newcommand{\pos}[1]{{\bf {\it {\texttt{p$_{#1}$}}}}}
\newcommand{\posnull}{{\bf {\it {\texttt{p$_{\emptyset}$}}}}}
\newcommand{\postext}[1]{{\bf {\it \texttt{p}$_{#1}$}}}
\newcommand{\unkpos}[1]{{\bf {\it {\texttt{\underline{unkpos}$_{#1}$}}}}}
\newcommand{\unkpostext}[1]{{\bf {\it \texttt{unkpos}$_{#1}$}}}

% experiment results
\newcommand{\bestbleu}{35.6}
\newcommand{\bestbleuunk}{37.5} % after unk translation
\newcommand{\bestbleuunkwmt}{36.6} % after unk translation
\newcommand{\bestunkimp}{2.8} % best of (bleuunk - bleu)
\newcommand{\unkimp}{1.9} % bestbleuunk - bestbleu
\newcommand{\unkimpilya}{2.7} % compare with ilya best system 34.8
\newcommand{\imprare}{4.8} % compare the last group of 500 sentences with the most rare words before and after unk translations (in Rarity Analysis)

%% Chapter 4
\newcommand{\correct}[1]{\textit{\color{blue} #1}}
\newcommand{\wrong}[1]{\textbf{\color{red} #1}}
\newcommand{\pt}{p_{t}} % context
\newcommand{\sotaold}{23.0} % WMT'14 English-German
\newcommand{\sotanew}{25.9} % WMT'15 English-German
\newcommand{\attngain}{5.0} % max improvement when using attention 
\newcommand{\localm}{local-m} % max improvement when using attention 
\newcommand{\localp}{local-p} % max improvement when using attention 
\DeclareMathOperator{\alignf}{align}
\DeclareMathOperator{\score}{score}

%% Chapter 5
\newcommand{\hide}[1]{}
\newcommand{\source}[1]{\biformat{#1}}
\newcommand{\close}[1]{\underline{\color{brown} #1}}
% char
\newcommand{\Ww}{\MB{W}} % W_h 
\newcommand{\Wc}{\MB{\breve{W}}} % W_c
\newcommand{\hc}{\MB{\breve{h}}_{t}} %h softmax char

\newcommand{\hybrid}{hybrid-NMT}
\newcommand{\rare}{$<${\it rare}$>$}

% best models
\newcommand{\modelword}{{\it (d)}}
\newcommand{\modelchar}{{\it (g)}}
\newcommand{\modelsmall}{{\it (k)}}
\newcommand{\model}{{\it (l)}}
\newcommand{\ensbleu}{20.7}
\newcommand{\gain}{2.1{-}11.4}
\newcommand{\chr}{chrF$_3$}

% chapter 6.1 multi-task
\newcommand{\ssl}{\emph{seq2seq}} % sequence to sequence learning
\newcommand{\mtssl}{\emph{MT-seq2seq}} % sequence to sequence learning
\newcommand{\mtl}{\texttt{MTL}} % sequence to sequence learning
\newcommand{\otm}{one-to-many}
\newcommand{\mto}{many-to-one}
\newcommand{\mtm}{many-to-many}




% -- Document --
\begin{document}

% Title
\title{Neural Machine Translation}
\author{Minh-Thang Luong}
\principaladviser{Christopher D. Manning}
\firstreader{Dan Jurafsky}
\secondreader{Andrew Ng}
\thirdreader{Quoc V. Le}

% Preface
\beforepreface
\input{chapters/preface}
\input{chapters/ack}
\afterpreface


% -- Sections --
\chapter{Introduction}
\label{c:intro}
\input{chapters/1-intro.tex}

\chapter{Background}
\label{c:background}
\input{chapters/2-background.tex}

\chapter{Copy Mechanisms}
\label{c:copy}
\input{chapters/3-copy.tex}

\chapter{Attention Mechanisms}
\label{c:attention}
\input{chapters/4-attention.tex}

\chapter{Hybrid Models}
\label{c:hybrid}
\input{chapters/5-hybrid.tex}

\chapter{The Future of NMT}
\label{c:future}
\input{chapters/6-future.tex}

\chapter{Conclusion}
\label{c:conclude}
\input{chapters/7-conclude.tex}

%\onlinesignature

\bibliographystyle{acl}
\bibliography{thesis}
\end{document}