\lecture{Jan.~22, 2009}{John Gubner}{ECE 729}{Information Theory}{John A. Gubner}


\chapter{The Sampling Theorem and Quantization}

\section{The Sampling Theorem}

Recall that a finite-energy waveform $x(t)$ is
{\bf bandlimited}\index{bandlimited waveform}
if its \bdx{Fourier transform} $X(f)$ satisfies
$X(f)=0$ for $|f|> W$ for some positive \bdx{bandwidth} $W$.
The \bdx{sampling theorem} says that such a waveform has
the representation
\bdm
x(t) = \sum_{k=-\infty}^\infty
x(k/f_s) \sinc(f_s [t-k/f_s]),
\edm
where the \bdx{sampling frequency} $f_s$ is at least
twice the bandwidth; i.e.,
$f_s \ge 2 W$.
The \bdx{sinc function} is
\bdm
\sinc(t) \de \left\{\begin{array}{cl}
\ds\frac{\sin(\pi t)}{\pi t}, & t\ne 0,
\\[2ex]
1, & t = 0.
\end{array}\right.
\edm

\begin{gtext}
The point of the sampling theorem is that knowledge of
the samples $x(k/f_s)$ allows us to reconstruct
$x(t)$ for all $t$.  Hence, in principle,
the problem of communicating bandlimited
waveforms has been reduced to the problem of communicating an
infinite sequence of real numbers.
\end{gtext}



\subsection*{Derivation}

To derive the sampling theorem,
we start with the fact that the assumptions on $x$ imply
\bdm
x(t) = \intii X(f) e^{j 2\pi f t} \,d f
=\int_{-W}^W X(f) e^{j 2\pi f t} \, d f.
\edm
Since $X$ is zero outside of $[-W,W]$,
$X$ is also zero outside of $[-f_s/2,f_s/2]$ for
any $f_s \ge 2 W$.
From the theory of Fourier series, we can write\footnote{
The sum is understood to converge in mean square.
}
\bdm
X(f) = \sum_{k=-\infty}^\infty c_k e^{-j 2\pi k f/f_s},
\quad |f| \le f_s/2,
\edm
where the Fourier-series coefficients $c_k$ are easily
found to be $c_k=x(k/f_s)/f_s$.
Now observe that\footnote{
The second integral can be viewed as an inner product,
and so the interchange of infinite sum and integral
is permissible by continuity of the inner product.
}
\beas
x(t) & = & \intii X(f) e^{j 2\pi f t}\, d f
\\
& = & \int_{-f_s/2}^{f_s/2} X(f) e^{j 2\pi f t}\, d f
\\
& = & \int_{-f_s/2}^{f_s/2} \biggl[\,
\sum_{k=-\infty}^\infty c_k e^{-j 2\pi k f/f_s}
\biggr]
e^{j 2\pi f t} \, d f
\\
& = & \sum_{k=-\infty}^\infty c_k
\int_{-f_s/2}^{f_s/2} e^{-j 2\pi k f/f_s} e^{j 2\pi f t} \,d f
\\
& = & \sum_{k=-\infty}^\infty x(k/f_s)/f_s
\cdot f_s \sinc(f_s[t-k/f_s]).
\eeas



\section{Quantization}


\subsection{Vector Quantizers}

Let $\bX=(X_1,\ldots,X_d)$ be an $\IR^d$-valued random vector.
An {\bfseries \itshape N\upshape-level vector
quantizer}\index{quantizer}\index{vector quantizer} is any function $q$ of the form\footnote{
Here $I_V$ denotes the \bdx{indicator function} of the set $V$; i.e.,
$I_V(\bx)\de 1$ if $\bx\in V$ and $I_V(\bx) \de 0$ otherwise.
}
\be
q(\bx) \de \sum_{i=1}^N \bc_i I_{V_i}(\bx),
\lb{qdef}
\ee
where the sets $V_1,\dots,V_N$ form a
partition of $\IR^d$ and the $\bc_i \in \IR^d$.
Recall that a \bdx{partition} is a collection of sets that
are pairwise disjoint and whose union is the whole space.

\subsubsection*{Optimizing the Partition -- Voronoi Regions}

If the $\bc_i$ are given, what is the best
choice of partition to minimize the mean-squared
error $\bE[\|\bX-q(\bX)\|^2]$?
Here $\|\cdot\|$ is the Euclidean norm on $\IR^d$.

We claim that no partition can do better than
the partition of {\bf Voronoi regions}\index{Voronoi region} defined
by\footnote{
To be precise, we should let $B_j$ denote the set in \er{Vjs}
and then put $V_1^\ast \de B_1$ and
$V_j^\ast \de B_j \cap B_{j-1}^c \cap \cdots \cap B_1^c$
for $j=2,\ldots,N$.
}
\be
V_j^\ast \de \{ \bx \in \IR^d : \|\bx-\bc_j\|^2 \le \|\bx-\bc_i\|^2
\mbox{ for all }i\, \}.
\lb{Vjs}
\ee
To establish this claim, let
\bdm
q^\ast(\bx) \de \sum_{j=1}^n \bc_j I_{V_j^\ast}(\bx).
\edm
Then write
\beas
\bE[\|\bX-q(\bX)\|^2] & = & \bE[\|\bX-q(\bX)\|^2 \cdot 1 \cdot 1 ]
\\
& = & \bE\biggl[ \|\bX-q(\bX)\|^2 \biggl(\,
\sum_{i=1}^N I_{V_i}(\bX)
\biggr)\biggl(\,
\sum_{j=1}^N I_{V_j^\ast}(\bX)
\biggr)\biggr]
\\
& = & \sum_{i=1}^N \sum_{j=1}^N
\bE[\|\bX-q(\bX)\|^2 I_{V_i}(\bX) I_{V_j^\ast}(\bX) ]
\\
& = & \sum_{i=1}^N \sum_{j=1}^N
\bE[\|\bX-\bc_i\|^2 I_{V_i}(\bX) I_{V_j^\ast}(\bX) ],
\eeas
since $q(\bx)\de \bc_i$ for $\bx\in V_i$.  On the other
hand, for $\bx\in V_j^\ast$, we know that
\bdm
\|\bx-\bc_j\|^2 \le \|\bx-\bc_i\|^2,
\quad\mbox{for all }i.
\edm
Hence,
\beas
\bE[\|\bX-q(\bX)\|^2] & \ge & \sum_{i=1}^N \sum_{j=1}^N
\bE[\|\bX-\bc_j\|^2 I_{V_i}(\bX) I_{V_j^\ast}(\bX) ]
\\
& = & \sum_{i=1}^N \sum_{j=1}^N
\bE[\|\bX-q^\ast(\bX)\|^2 I_{V_i}(\bX) I_{V_j^\ast}(\bX) ]
\\
& = & \bE\biggl[ \|\bX-q^\ast(\bX)\|^2 \biggl(\,
\sum_{i=1}^N I_{V_i}(\bX)
\biggr)\biggl(\,
\sum_{j=1}^N I_{V_j^\ast}(\bX)
\biggr)\biggr]
\\
& = & \bE[\|\bX-q^\ast(\bX)\|^2].
\eeas
