\lecture{Jan.~25, 2009}{John Gubner}{ECE 729}{Information Theory}{John A. Gubner}


\section*{Lecture Outline}
\begin{itemize}
\item Finish Chapter 2: Optimizing the coefficients $\bc_i$ when
the partition sets $V_j$ are fixed.  Then introduce the Lloyd--Max Algorithm.
\item Start Chapter 3: Data Compression Using Variable-Length Codes.
\end{itemize}


\subsubsection*{Optimizing the Coefficients -- Linear Estimation}

If the sets $V_i$ of an arbitrary quantizer $q$ in \er{qdef} are given,
what are the best $\bc_i$ to use?

Let $\bY \de [ I_{V_1}(\bX),\ldots,I_{V_N}(\bX)]^\T$.  Then
$q(\bX) = A \bY$, where
$A$ is the $d\crs N$ matrix $A \de [\bc_1,\ldots,\bc_N]$.
Hence, the mean-squared error is
$\bE[\|X-A \bY\|^2]$.  In other words, finding the best
vectors $\bc_i$ is equivalent to finding the best linear
transformation (matrix) $A$.  By the
\bdx{orthogonality principle} \cite[\href{http://books.google.com/books?id=pa20eZJe4LIC&pg=PA344&dq=gubner}{Section~8.4}]{Gubner},
the best matrix $A$ is any
solution of the \bdx{normal equations}
\be
A R_{\bY} = R_{\bX \bY},
\lb{neq}
\ee
where $R_{\bY}$ is the correlation (not covariance) matrix
$R_{\bY} \de \bE[\bY \bY^\T]$ and $R_{\bX \bY}$ is the cross-correlation
matrix $R_{\bX \bY} \de \bE[\bX \bY^\T]$.

\subsection{The Lloyd--Max Algorithm}

The repeated alternating iteration of optimizing the partition
and $\bc_i$ is the {\bf Lloyd--Max algorithm} \cite{Lloyd,Max}.
The scalar case,
$d=1$, is quite simple.  First, if $c_1 < \cdots < c_N$,
then \er{Vjs} reduces to
\bdm
V_j^\ast = \Bigl( \frac{c_{j-1}+c_j}{2} , \frac{c_j+c_{j+1}}{2} \Bigr].
\edm
Second, since $R_Y$ is diagonal, \er{neq} is easily solved to yield
\bdm
c_j^\ast = \frac{\bE[X I_{V_j}(X)]}{\bP(X\in V_j)}.
\edm
If $X$ has density function $f$, and if $V_j$ is an interval,
say $V_j=(a_j,b_j]$, then
\bdm
c_j^* = \frac{\ds\int_{a_j}^{b_j} x f(x) \, d x}{
\ds\int_{a_j}^{b_j} f(x) \, d x}.
%= \frac{\ds\int_{a_j}^{b_j} x f(x) \, d x}{F(b_j)-F(a_j)},
\edm
%where $F$ denotes the cumulative distribution function of $X$.

\bigskip

\chapter{Data Compression Using Variable-Length Codes}

Consider an \bdx{alphabet} of source letters, say
$\sX = \{\sla,\slb,\slc,\sld\}$.  Here are possible choices
for a fixed-length binary code and a variable-length binary code.
\begin{center}
\begin{tabular}{c|c}
symbol & codeword
\\
\hline
\sla & 00
\\
\slb & 01
\\
\slc & 10
\\
\sld & 11
\end{tabular}
\hfil
\begin{tabular}{c|c}
symbol & codeword
\\
\hline
\sla & \makebox[2em][l]{0}
\\
\slb & \makebox[2em][l]{10}
\\
\slc & \makebox[2em][l]{110}
\\
\sld & \makebox[2em][l]{111}
\end{tabular}
\end{center}
The expected codeword length of the fixed-length
code is always $2$.  If the source letters are equally
likely, then the expected codeword length of the
variable-length code is $9/4=2.25$.  However, if
$p(\sla)=1/2$ and $p(\slb)=p(\slc)=p(\sld)=1/6$, then the expected
length of the variable-length code is $11/6=1.8\overline{3}$.

\medskip

Goals:
\begin{itemize}
\item
Characterize unique decodability or self-punctuating codes.
\item
Characterize the minimum possible expected legnth.
This result depends on the entropy of the source.
\item
Develop an algorithm for finding a variable-length
code whose expected length is the minimum possible.
\end{itemize}


\section{Codewords}
Let $\sD$ be a finite set containing $D$ elements.
Without loss of generality, take $\sD \de \{0,1,\ldots,D-1\}$.
The set $\sD$ is called the \bdx{codeword alphabet}.
Let $\sD^*$ denote the set of al finite-length strings
from $\sD$.  For $\bd \in \sD^*$,
we let $\ell(\bd)$ denote the number of letters in $\bd$.
For example, if $D=3$, $\sD=\{0,1,2\}$, and if
$\bd = 0020112$, then $\ell(\bd) = 7$.
Our {\bf codewords}\index{codeword} will be elements of $\sD^*$.



\section{Codes}

Let $\sX$ be an arbitrary set of \bdx{source symbols}.
Let $|\sX|$ denote the \bdx{cardinality} of $\sX$; i.e.,
$|\sX|$ is the number of elements in $\sX$.  A function
or mapping $C \colon \sX \to \sD^*$ is called a
\bdx{source code}.  The \bdx{range} of $C$ is the set
$\{ C(x) : x\in \sX\}$.
In coding theory, the range of the function $C$ is called
the \bdx{codebook}.  Typically, the codebook is always a
proper subset of $\sD^*$.

In this course, we always work with finite or countably infinite
sets $\sX$.  In this case, we assume that we are given
a \bdx{probability mass function} (pmf) $p$ on $\sX$; i.e.,
$p(x) \ge 0$ and $\sum_{x\in \sX} p(x)=1$.  When $p$ is given,
the \bdx{expected length} of the source code $C$ is
\bdm
L(C) \de \sum_{x\in \sX} \ell(C(x)) p(x).
\edm
In other words, if $X$ is an $\sX$-valued \bdx{random variable}
(RV) with pmf $p(x)=\bP(X=x)$, then
\bdm
\bE[\ell(C(X))] = \sum_{x\in \sX} \ell(C(x)) \bP(X=x)
=\sum_{x\in \sX} \ell(C(x))p(x) = L(C).
\edm


\begin{thebibliography}{0}


\bibitem{Gubner}
\href{http://books.google.com/books?id=pa20eZJe4LIC&printsec=frontcover&dq=gubner&cd=1#v=onepage&q=&f=false}{J. A. Gubner,
{\it Probability and Random Processes for
Electrical
and Computer Engineers}.
Cambridge, UK: Cambridge Univ.\ Press, 2006.}

\bibitem{Lloyd}
\href{http://ieeexplore.ieee.org.ezproxy.library.wisc.edu/search/srchabstract.jsp?arnumber=1056489&isnumber=22725&punumber=18&k2dockey=1056489@ieeejrns&query=%28%28least+squares+quantization+in+pcm%29%3Cin%3Eti+%29&pos=0&access=no}{S. P. Lloyd,
``Least squares quantization in PCM,''
unpublished Bell Laboratories Memorandum, July 31, 1957;
also
{\it IEEE Trans.\ Inform.\ Theory},
vol.~IT-28, pp.~129--137, Mar.~1982.}

\bibitem{Max}
\href{http://ieeexplore.ieee.org.ezproxy.library.wisc.edu/search/searchresult.jsp?query1=quantizing+for+minimum+distortion&scope1=ti&op1=and&query2=&scope2=metadata&op2=and&query3=&scope3=metadata&queryText=+%28%28quantizing+for+minimum+distortion%29%3Cin%3Eti+%29+&history=yes&reqloc=basic&submit=Run+Search&queryblock=&srchlist=publist&coll1=ieeejrns&coll2=ieejrns&coll9=aip&coll3=ieeecnfs&coll4=ieecnfs&coll5=ieeestds&coll6=preprint&coll7=books&coll8=modules&std_status=all&currweek=19-Jan-2010&srchyr=allyr&py1=1950&py2=2010&disp=cit&maxdoc=100&ResultCount=25&SortField=Score&SortOrder=desc}{J. Max,
``Quantizing for minimum distortion,''
{\it IRE Trans.\ Inform.\ Theory},
vol.~IT-6, pp.~7--12, Mar.~1960.}

\end{thebibliography}