Caught up Coding

CodingAndCryptography/00_modelling_communication.tex

@@ -17,7 +17,7 @@ \section{Modelling communication}
 Noiseless coding is adapted to the source.
 
 Here is an example of noisy coding.
-Each book has an ISBN $a_1 a_2 \dots a_9 a_{10}$ where the $a_1, \dots, a_9$ are digits in $\qty{0, \dots, 9}$, and $a_{10} \in \qty{0, \dots, 9, X}$ such that $11 \mid \sum_{j=1}^{10} j a_j$.
+Each book has an ISBN $a_1 a_2 \dots a_9 a_{10}$ where the $a_1, \dots, a_9$ are digits in $\qty{0, \dots, 9}$, and $a_{10} \in \qty{0, \dots, 9, X}$ s.t. $11 \mid \sum_{j=1}^{10} j a_j$.
 This coding system detects the common human errors of writing an incorrect digit and transposing two adjacent digits.
 Noisy coding is adapted to the channel, which in this case is the human reading the number and typing it into a computer.
 
@@ -68,7 +68,7 @@ \section{Modelling communication}
 \end{definition}
 
 \begin{definition}[Capacity]
-    A channel can \vocab{transmit reliably at rate $R$} if there is a sequence of codes $(C_n)_{n=1}^\infty$ with each $C_n$ a code of length $n$ such that $\lim_{n \to \infty} \rho(C_n) = R$ and $\lim_{n \to \infty} \hat e(C_n) = 0$.
+    A channel can \vocab{transmit reliably at rate $R$} if there is a sequence of codes $(C_n)_{n=1}^\infty$ with each $C_n$ a code of length $n$ s.t. $\lim_{n \to \infty} \rho(C_n) = R$ and $\lim_{n \to \infty} \hat e(C_n) = 0$.
     The \vocab{capacity} of a channel is the supremum of all reliable transmission rates.
 \end{definition}
 

CodingAndCryptography/01_noiseless_coding.tex

@@ -120,8 +120,8 @@ \subsection{McMillan's inequality}
 
 \subsection{Entropy}
 \vocab{Entropy} is a measure of `randomness' or `uncertainty' in an input message.
-Suppose that we have a random variable $X$ taking a finite number of values $x_1, \dots, x_n$ with probability $p_1, \dots, p_n$.
-Then, the entropy of this random variable is the expected number of fair coin tosses required to determine $X$.
+Suppose that we have a r.v. $X$ taking a finite number of values $x_1, \dots, x_n$ with probability $p_1, \dots, p_n$.
+Then, the entropy of this r.v. is the expected number of fair coin tosses required to determine $X$.
 
 \begin{example}
     Suppose $p_1 = p_2 = p_3 = p_4 = \frac{1}{4}$.
@@ -136,7 +136,7 @@ \subsection{Entropy}
 In a sense, the first example is `more random' than the second, as its entropy is higher.
 
 \begin{definition}[Entropy]
-    The \vocab{entropy} of a random variable $X$ taking a finite number of values $x_1, \dots, x_n$ with probabilities $p_1, \dots, p_n$ is defined to be
+    The \vocab{entropy} of a r.v. $X$ taking a finite number of values $x_1, \dots, x_n$ with probabilities $p_1, \dots, p_n$ is defined to be
     \begin{align*}
         H(X) = H(p_1, \dots, p_n) = -\sum_{i=1}^n p_i \log p_i = -\expect{\log p_i}
     \end{align*}
@@ -195,7 +195,7 @@ \subsection{Gibbs' inequality}
 
 \subsection{Optimal codes}
 Let $\mathcal A = \qty{\mu_1, \dots, \mu_m}$ be an alphabet of $m \geq 2$ messages, and let $\mathcal B$ be an alphabet of length $a \geq 2$.
-Let $X$ be a random variable taking values in $A$ with probabilities $p_1, \dots, p_m$.
+Let $X$ be a r.v. taking values in $A$ with probabilities $p_1, \dots, p_m$.
 
 \begin{definition}[Optimal Code]
     A code $c \colon \mathcal A \to \mathcal B^\star$ is called \vocab{optimal} if it has the smallest possible expected word length $\sum p_i \ell_i = \expect{S}$ among all decipherable codes.
@@ -350,7 +350,7 @@ \subsection{Huffman coding}
     If $m = 2$, then the codewords are 0 and 1, which is clearly optimal.
 
     Assume $m > 2$, and let $c_m$ be the Huffman code for $X_m$ which takes values $\mu_1, \dots, \mu_m$ with probabilities $p_1 \geq \dots \geq p_m$.
-    $c_m$ is constructed from a Huffman code $c_{m-1}$ with random variable $X_{m-1}$ taking values $\mu_1, \dots, \mu_{n-2}, \nu$ with probabilities $p_1, \dots, p_{m-2}, p_{m-1} + p_m$.
+    $c_m$ is constructed from a Huffman code $c_{m-1}$ with r.v. $X_{m-1}$ taking values $\mu_1, \dots, \mu_{n-2}, \nu$ with probabilities $p_1, \dots, p_{m-2}, p_{m-1} + p_m$.
     The code $c_{m-1}$ is optimal by the inductive hypothesis.
     The expected word length $\expect{S_m}$ is given by
     \begin{align*}
@@ -384,16 +384,16 @@ \subsection{Huffman coding}
 \end{remark}
 
 \subsection{Joint entropy}
-Let $X, Y$ be random variables with values in $\mathcal A, \mathcal B$.
-Then, the pair $(X, Y)$ is also a random variable, taking values in $\mathcal A \times \mathcal B$.
+Let $X, Y$ be r.v.s with values in $\mathcal A, \mathcal B$.
+Then, the pair $(X, Y)$ is also a r.v., taking values in $\mathcal A \times \mathcal B$.
 This has entropy $H(X,Y)$, called the \vocab{joint entropy} for $X$ and $Y$.
 \begin{align*}
     H(X,Y) = - \sum_{x \in \mathcal A} \sum_{y \in \mathcal B} \prob{X = x, Y = y} \log \prob{X = x, Y = y}
 \end{align*}
-This construction generalises to finite tuples of random variables.
+This construction generalises to finite tuples of r.v.s.
 
 \begin{lemma}
-    Let $X, Y$ be random variables taking values in $\mathcal A, \mathcal B$.
+    Let $X, Y$ be r.v.s taking values in $\mathcal A, \mathcal B$.
     Then $H(X,Y) \leq H(X) + H(Y)$, with equality iff $X$ and $Y$ are independent.
 \end{lemma}
 

CodingAndCryptography/02_noisy_channels.tex

@@ -210,11 +210,11 @@ \subsection{Covering estimates}
 \end{proof}
 
 \begin{definition}[Perfect Code]
-    An $e$-error correcting code $C$ of length $n$ such that $\abs{C} = \frac{2^n}{V(n,e)}$ is called \vocab{perfect}.
+    An $e$-error correcting code $C$ of length $n$ s.t. $\abs{C} = \frac{2^n}{V(n,e)}$ is called \vocab{perfect}.
 \end{definition}
 
 \begin{remark}
-    Equivalently, a code is perfect if for all $x \in \mathbb F_2^n$, $\exists! \; c \in C$ such that $d(x,c) \leq e$.
+    Equivalently, a code is perfect if for all $x \in \mathbb F_2^n$, $\exists! \; c \in C$ s.t. $d(x,c) \leq e$.
     Alternatively, $\mathbb F_2^n$ is a union of disjoint balls $B(c,e)$ for all $c \in C$, or that any collection of $e + 1$ will cause the message to be decoded incorrectly.
 \end{remark}
 
@@ -254,7 +254,7 @@ \subsection{Covering estimates}
 
 \begin{proof}
     Let $m = A(n,d+1)$, and let $C$ be an $[n,m,d+1]$-code.
-    Let $c_1, c_2 \in C$ be distinct codewords such that $d(c_1,c_2) = d+1$.
+    Let $c_1, c_2 \in C$ be distinct codewords s.t. $d(c_1,c_2) = d+1$.
     Let $c_1'$ differ from $c_1$ in exactly one of the places where $c_1$ and $c_2$ differ.
     Then $d(c_1', c_2) = d$.
     If $c \in C$ is any codeword not equal to $c_1$, then $d(c,c_1) \leq d(c,c_1') + d(c_1',c_1)$ hence $d + 1 \leq d(c,c_1') + 1$, so the code given by $C \cup \qty{c_1'} \setminus \qty{c_1}$ has minimum distance $d$, but has length $n$ and size $m$.
@@ -265,7 +265,7 @@ \subsection{Covering estimates}
     Equivalently, $A(n,d) = \max \qty{m : \exists [n,m,d'] \text{-code, for some } d' \geq d}$.
 \end{corollary}
 
-\begin{theorem}
+\begin{theorem} ~\vspace*{-1.5\baselineskip}
     \begin{align*}
         \frac{2^n}{V(n,d-1)} \leq A(n,d) \leq \frac{2^n}{V\qty(n,\floor*{\frac{d-1}{2}})}
     \end{align*}
@@ -285,6 +285,7 @@ \subsection{Covering estimates}
     \end{align*}
     as required.
 \end{proof}
+
 \begin{example}
     Let $n = 10, d = 3$.
     Then $V(n,1) = 11$ and $V(n,2) = 56$, so the GSV bound is $\frac{2^{10}}{56} \leq A(10,3) \leq \frac{2^{10}}{11}$.
@@ -305,19 +306,20 @@ \subsection{Asymptotics}
     \end{enumerate}
     where $H(\delta) = -\delta \log \delta - (1-\delta)\log (1-\delta)$.
 \end{proposition}
+
 \begin{proof}
-    \vocab{(i) implies (ii).}
+    \emph{(i) implies (ii).}
     By the GSV bound, we find
     \begin{align*}
         A(n,\floor*{n\delta}) \geq \frac{2^n}{V(n,\floor*{n\delta} - 1)} \geq \frac{2^n}{V(n,\floor*{n\delta})}
     \end{align*}
-    Taking logarithms,
+    Taking logarithms and dividing by $n$,
     \begin{align*}
         \frac{1}{n}\log A(n,\floor*{n\delta}) \geq 1 - \frac{\log V(n,\floor*{n\delta})}{n} \geq 1 - H(\delta)
     \end{align*}
-    \vocab{Part (i).}
+    \emph{Part (i).}
     $H(\delta)$ is increasing for $\delta < \frac{1}{2}$.
-    Therefore, without loss of generality, we may assume $n\delta$ is an integer.
+    Therefore, wlog, we may assume $n\delta$ is an integer.
     Now, as $\frac{\delta}{1-\delta} < 1$,
     \begin{align*}
         1 &= (\delta + (1-\delta))^n \\
@@ -333,37 +335,59 @@ \subsection{Asymptotics}
     \end{align*}
     as required.
 \end{proof}
+
 The constant $H(\delta)$ in the proposition is optimal.
+
 \begin{lemma}
     $\lim_{n \to \infty} \frac{\log V(n,\floor*{n\delta})}{n} = H(\delta)$.
 \end{lemma}
+
 \begin{proof}
-    Exercise.
-    Follows from Stirling's approximation to factorials.
+    Wlog assume $0 < \delta < \frac{1}{2}$.
+    Let $0 \leq r \leq \frac{n}{2}$.
+    Recall $V(n, r) = \sum_{i=0}^{r} \binom{n}{i}$.
+    Then
+    \begin{align*}
+        \binom{n}{r} \leq V(n, r) \leq (r + 1) \binom{n}{r} \quad (\ast)
+    \end{align*}
+    Recall Stirling's formula: $\ln n! = n \ln n - n + O(\log n)$.
+    \begin{align*}
+        \ln \binom{n}{r} &= (n \ln n - n) - (r \ln r - r) \\ &- \qty((n - r) \log(n - r) - (n - r)) + O(\log n) \\
+        \log \binom{n}{r} &= -r \log \frac{r}{n} - (n - r) \log \frac{n - r}{n} + O(\log n) \\
+        &= n H\qty(\frac{r}{n}) + O(\log n).
+        \intertext{By $(\ast)$}
+        H\qty(\frac{r}{n}) + O\qty(\frac{\log n}{n}) &\leq \frac{\log V(n, r)}{n} \leq H\qty(\frac{r}{n}) + O\qty(\frac{\log n}{n}) \\
+        \lim\limits_{n \to \infty} \frac{\log V(n \floor{n \delta})}{n} &= H(\delta)
+    \end{align*}
 \end{proof}
 
 \subsection{Constructing new codes from old}
 Let $C$ be an $[n,m,d]$-code.
-\begin{example}
+
+\begin{example}[Parity Check Extension]
     The \vocab{parity check extension} is an $[n+1,m,d']$-code given by
     \begin{align*}
-        C^+ = \qty{\qty(c_1, \dots, c_n, \sum_{i=1}^n c_i) \midd (c_1, \dots, c_n) \in C}
+        C^+ = \qty{\qty(c_1, \dots, c_n, \sum_{i=1}^n c_i \mod 2) : (c_1, \dots, c_n) \in C}
     \end{align*}
     where $d'$ is either $d$ or $d + 1$, depending on whether $d$ is odd or even.
 \end{example}
-\begin{example}
+
+\begin{example}[Punctured Code]
     Let $1 \leq i \leq n$.
     Then, deleting the $i$th digit from each codeword gives the \vocab{punctured code}
     \begin{align*}
-        C^- = \qty{(c_1, \dots, c_{i-1}, c_{i+1}, \dots, c_n) \midd (c_1, \dots, c_n) \in C}
+        C^- = \qty{(c_1, \dots, c_{i-1}, c_{i+1}, \dots, c_n) : (c_1, \dots, c_n) \in C}
     \end{align*}
     If $d \geq 2$, this is an $[n-1, m, d']$-code where $d'$ is either $d$ or $d - 1$.
 \end{example}
-\begin{example}
-    Let $1 \leq i \leq n$ and let $\alpha \in \mathbb F_2$.
+
+\begin{example}[Shortened Code]
+    Fix $1 \leq i \leq n$ and $\alpha \in \mathbb F_2$.
     The \vocab{shortened code} is
     \begin{align*}
-        C' = \qty{(c_1, \dots, c_{i-1}, c_{i+1}, \dots, c_n) \midd (c_1, \dots, c_{i-1}, \alpha, c_{i+1}, \dots, c_n) \in C}
+        C' = \qty{(c_1, \dots, c_{i-1}, c_{i+1}, \dots, c_n) : (c_1, \dots, c_{i-1}, \alpha, c_{i+1}, \dots, c_n) \in C}
     \end{align*}
     This is an $[n-1,m',d']$ with $d' \geq d$ and $m' \geq \frac{m}{2}$ for a suitable choice of $\alpha$.
+
+    Note that puncturing and shortenings are not the same thing.
 \end{example}