lecture-14.tex

\documentclass[a4paper,10pt,english]{article}
\input{header}
%opening
\title{Lecture 14: Foster's Stability Criterion}
\date{}%01 Mar 2016}
\author{}

\begin{document}
\maketitle
\section{Foster's theorem for Markov chains}
For many Markov chains, finding stationary distribution is impossible. 
At the least, we would like to know if they are positive recurrent. 

\begin{lem} 
An irreducible Markov chain is positive recurrent iff there exists a state $i \in E$ such that
\begin{align*}
\lim_{N \to \infty} \E\Big[ \sum_{n=1}^{N} \boldsymbol{1}_{\{X_n=i \}} \Big] > 0. % \Longleftrightarrow x \ is \ positive \ recurrent  
\end{align*}
\end{lem}
\begin{proof}
For an irreducible Markov chain, it suffices to show that there exists a positive recurrent state $i \in E$. 
By renewal theory, for any state $i \in E$ of a Markov chain $X$,
\begin{align*}
\lim_{N \to \infty} \E\Big[ \sum_{n=1}^{N} \boldsymbol{1}_{\{X_n=i\}} \Big] = \frac{1}{\mu _{ii}} 
\end{align*}
where $\mu_{ii}$ is the mean return time to state $i$, 
\eq{
\mu _{ii}=
\begin{cases}
	\infty, & i\text{ transient},\\
	\sum_{m \in \N}m f^{m}_{ii} &i \text{ recurrent}.
\end{cases}
}
This implies that a state $i \in E$ being positive recurrent is equivalent to the given hypothesis $1/\mu_{ii} > 0$. 
%\begin{align*}
%\lim_{N \to \infty} \E\Big[ \sum_{n=1}^{N} \boldsymbol{1}_{\{X_n=i \}} \Big] > 0. % \Longleftrightarrow x \ is \ positive \ recurrent  
%\end{align*}
\end{proof}

The following theorem gives a sufficient criterion for the positive recurrence of a Markov chain in terms of a Lyapunov function $L$. 
The value $L(i)$ for any state $i$ attained by Markov chain denotes ``energy'' or ``potential'' of that state. 
The idea is that if the mean energy decreases for all but finitely many states, the Markov chain keeps returning to a finite set of states. 
That is, the Markov chain is positive recurrent. 
For any positive function $g: E \to \R_+$ and threshold $k$, we can define level sets 
\eq{
F(g,k) &= \{ i\in E: g(i) \leq k\}.
}
\begin{thm}[F. G. Foster,1950]
Let $X = \{X_n: n \in \N_0\}$ be an irreducible Markov chain on countable state space $E$. 
If there exist a function $L : E \to \R_+$ with bounded mean $\E L(X_0) < \infty$, 
such that for some $K, k \geq 0$ and $\epsilon>0$ the following conditions hold
\begin{enumerate}[i\_]
\item \textbf{Unbounded growth of energy.} $F(L,k)$ is a finite set.
\item \textbf{Bounded energy on bounded set.} $\E[L(X_n)|X_{n-1}] < K$,when $X_{n-1}) \in F(L,k)$.
\item \textbf{Negative drift on unbounded set.} $\E[L(X_n)-L(X_{n-1})|X_{n-1}] < -\epsilon$ if $X_{n-1})\notin F(L,k)$
\end{enumerate}
Then the Markov chain $X$ is positive recurrent. 
%(L $\equiv$ "potential function" or energy or Lyapunov function).
\end{thm}
\begin{proof}
%Since the Markov chain is irreducible, iconsider
For the Markov chain $X$ and the Lyapunov function $L$, choose $K,k$ and $\epsilon$ such that the hypotheses hold.  
We refer to the finite set of states $F(L,k)$ as $F$. 
%\eq{
%F &= \{i \in E: L(i) \leq k\}.
%}
We can write the following using telescopic sum
\eq{
\E L(X_n) - \E L(X_0) %&=  \sum_{n=1}^{N}\E[L(X_n)-L(X_{n-1})] 
&=  \sum_{n=1}^{N}\E[L(X_n)-L(X_{n-1})1\{X_{n-1} \notin F \}] + \sum_{n=1}^{N}\E[L(X_n)-L(X_{n-1}) 1\{X_{n-1} \in F\}].
}
Using the second and third hypotheses, we can write 
\eq{
\E L(X_n) - \E L(X_0) &\leq -\sum_{n=1}^{N}\epsilon\E1\{X_{n-1} \notin F \} + \sum_{n=1}^{N}K \E1\{X_{n-1} \in F\}
= -\epsilon N + \sum_{n=1}^{N}(K+\epsilon) \E1\{X_{n-1} \in F\}.
}
%\begin{align*}
%& 0 \leq \E[L(X_n)] = \E[L(X_0)] +   \sum_{n=1}^{N}\E[L(X_n)-L(X_{n-1})]\\
%&= \E[L(X_0)] + \sum_{n=1}^{N}\E[L(X_n)-L(X_{n-1})]\boldsymbol{1}_{\{L(X_{n-1})>k\}} + \sum_{n=1}^{N}\E[L(X_n)-L(X_{n-1})]\boldsymbol{1}_{\{L(X_{n-1})\leq k\}}\\
%&\leq \E[L(X_0)] + \sum_{n=1}^{N}\E[-\epsilon \        \boldsymbol{1}_{\{L(X_{n-1})>k\}}] + \sum_{n=1}^{N}\E[K \ \boldsymbol{1}_{\{L(X_{n-1})\leq k\}}]\\
%&\to \Bigg[\E\sum_{n=1}^{N} \boldsymbol{1}_{\{L(X_{n-1})\leq k\}}\Bigg](K + \epsilon) \geq -\E[L(X_0)] + \epsilon N\\\\
%&\to \frac{1}{N} \Bigg[\E\sum_{n=1}^{N} \boldsymbol{1}_{\{L(X_{n-1})\leq k\}}\Bigg] \geq -\frac{\E[L(X_0)]}{N(K+\epsilon)} + \frac{\epsilon}{K+\epsilon}\\\\
%&\to \limsup_{N\to \infty}\frac{1}{N} \Bigg[\E\sum_{n=1}^{N} \boldsymbol{1}_{\{L(X_{n-1})\leq k\}}\Bigg] \geq \frac{\epsilon}{K+\epsilon}
%\end{align*}
Since $L$ is a positive function, we can re-arrange the above equation to write 
\eq{
\frac{1}{N}\E\sum_{n=1}^{N}1\{X_{n-1} \in F\} &= \sum_{i \in F}\left[\frac{1}{N}\E\sum_{n=1}^{N}1\{X_{n-1} = i\}\right] \geq \frac{\epsilon}{K + \epsilon} - \frac{\E L(X_0)}{N(K+\epsilon)}
}
Taking limit superior as $N \uparrow \infty$ on both sides, we obtain
\eq{
\sum_{i \in F}\left[\lim\sup_{N \to \infty}\frac{1}{N}\E\sum_{n=1}^{N}1\{X_{n-1} = i\}\right] & \geq \frac{\epsilon}{K + \epsilon}.
}
%Let $F = \{x \in \N_0: L(x) \leq k\}, \ |F| < \infty$. Now,
%\begin{align*}
%& \limsup_{N\to \infty}\frac{1}{N} \Bigg[\E\sum_{n=1}^{N} \boldsymbol{1}_{\{L(X_{n-1})\leq k\}}\Bigg]\\\\
%&=\limsup_{N\to \infty}\frac{1}{N} \Bigg[\E\sum_{n=1}^{N}\sum_{x \in F} \boldsymbol{1}_{\{X_{n-1}\leq k\}}\Bigg] \leq \sum_{x \in F} \Bigg[ \limsup_{N\to \infty}\frac{1}{N} \E\sum_{n=1}^{N} \boldsymbol{1}_{\{X_{n-1}\leq k\}}\Bigg]\\\\ 
%&=\sum_{x \in F} \Bigg[ \limsup_{N\to \infty}\frac{1}{N} \E\sum_{n=1}^{N} \boldsymbol{1}_{\{X_{n-1}\leq k\}}\Bigg] \geq \frac{\epsilon}{K+\epsilon}>0
%\end{align*}
Therefore, there exist some $x \in F$ such that 
\eq{
\limsup_{N\to \infty}\frac{1}{N} \E\Bigg[ \sum_{n=1}^{N} \boldsymbol{1}_{\{X_{n-1}\leq k\}}\Bigg] \geq \frac{\epsilon}{(K+\epsilon)|F|}>0.
}
%Therefore there exist some $x \in F$ such that ,
%\[\lim_{N\to \infty}\frac{1}{N} \E\Bigg[\sum_{n=1}^{N} \boldsymbol{1}_{\{X_{n-1}\leq k\}}\Bigg] >0\]
\end{proof}
\section{Applications: max-weight queue scheduling}
Consider $N$ discrete time queues served by a single server as shown in Figure~\ref{Fig:SharedServerQueue}. 
In each queue $i \in [N]$, the number of existing customers, the number of customer arrivals, and the available service are denoted by $X_i(n), A_i(n)$, and $R_i(n)$ respectively  at each discrete time slot $n \in \N$. 
Arrival $A_i(n) \in \N_0$ at each queue $i \in [N]$ at each time $n$ is assumed to be independent, 
and it satisfies the following conditions, 
\begin{xalignat*}{5}
&\E A_i(n) = \lambda _i,&& P\{A_i(n)=0\} > 0,&& \E A_i(n)^2 \leq C.
\end{xalignat*}
This implies mean rate of arrival to the system is $\sum_{i\in[N]}\lambda_i$. 
\begin{figure}
\centering
\includegraphics[scale=0.5]{Figures/saa.png}
\caption{N queues single server, server chooses one queue and serve upto one packet}
\label{Fig:SharedServerQueue}
\end{figure}
The server is shared by all $N$ queues, and can only serve one customer in any one queue in any time slot $n$. 
%The server can serve only one customer at each time $n$. 
The customer leaves the queue if served. 
Hence, the maximum rate of departure from all the queues is unity. 
We will assume that aggregate mean rate of arrival to the system is smaller than maximum available service to the system, that is $\sum_{i \in [N]}\lambda_i < 1$. 
Let $I(n) \in [N]$ be the queue index served by the server in each time $n$, 
then the service available to queue $i$ at time $n$ is $R_i(n) = 1\{I(n) = i\}$. 
%Server picks one queue $Q(t) \in [N]$ for service. Let $R_i(t) =  \boldsymbol{1}\{Q(t) = i\}$. 
%One packet is served from $Q(t)$ if it is not empty. 
%Let $X_i(t)$ = number of packets in queue $i$ just before time slot $t$.\\
We can write the time evolution for queue $i$ as
\eq{
X_i(n+1) = (X_i(t) + A_i(t) - R_i(t))_+,
}
where $a_+ = \max(0,a)$. 

At each time $n$, the selection of queue $I(n)$ to be served is called \textbf{scheduling}.  
Consider a scheduling policy  based solely on the current state of all queues $X(n) = (X_1(n), \dots, X_N(n))$. 
For such policies, the aggregate state of the queues form a Markov chain, 
since each arrival is independent, and queue evolution only depends on the current state. 
This is a difficult Markov chain to study if the scheduling policy is not random. 
We are interested in a scheduling policy that would make the Markov chain positive recurrent if $\sum_{i \in [N]}\lambda_i < 1$. 
%aslo \[X_i(t+1) = X_i(t) + A_i(t) - R_i(t) + L_i(t)       \]Where 
%\begin{equation*}
%L_i(t) = 
%\begin{cases}
%  1 \ \ \ \,service \ attempted \ when \ i \ is \ empty\\
%  0 \ \ \ \,otherwise 
%\end{cases}
%\end{equation*}
%Mean rate of arrivals to system : $= \sum_{i=1}^{N} \lambda _i$\\
%Maximum rate of departure $= 1$, we will assume $ \sum_{i=1}^{N} \lambda _i < 1$.
\begin{thm}[max-weight scheduling algorithm, 1992]
Max-weight scheduling policy selects the longest queue for service at each time $n$, 
\eq{
I(n) = \arg \max\{i \in N: X_i(n)\}.
}   
Under this policy, $X(n) = (X_1(n), \dots, X_N(n))$ is an irreducible and aperiodic Markov chain on state space $\N_0^N$. 
If $\sum_{i \in [N]} \lambda _i < 1$, then Markov chain $X$ is positive recurrent. 
\end{thm}
\begin{proof}
Let $\sum_{i \in [N]}\lambda_i < 1 - \delta$ for some $\delta > 0$. 
For Foster's theorem, we define a Lyapunov function
\eq{
L(x) &= \frac{1}{2} \sum_{i=1}^{N}x_i^2.
}
We consider the difference in Lyapunov function between two successive states of the Markov chain. 
From the queue evolution equation and since $a_+^2 \leq a^2$, we have
\begin{align*}
L(X(n+1)) - L(X(n)) %&= \frac{1}{2} \sum_{i=1}^{N}\Big[ (X_i(t))^2 - (X_i(t-1))^2 \Big]\\
&\leq %\frac{1}{2} \sum_{i \in [N]}\left[ (X_i(n) + A_i(n) - R_i(n))^2 - (X_i(n))^2 \right] = 
%& \leq \frac{1}{2} \sum_{i=1}^{N}\Big[ \{X_i(t-1) + A_i(t-1) - R_i(t-1)\}^2 - (X_i(t-1))^2 \Big]
\sum_{i \in [N]}\left[X_i(n)(A_i(n)-R_i(n)) + \frac{1}{2}(A_i(n)-R_i(n))^2\right].
\end{align*}
Since $X_i(n)R_i(n) = X_i(n)1\{I(n) = i\}$ where $I(n) = \arg\max\{j \in [N]: X_j(n)\}$, we have
\eq{
\sum_{i \in [N]}X_i(n)R_i(n) = \max\{X_i(n): i \in [N]\}.
}
Therefore, we can bound the conditional expectation of the drift
\eq{
\E\left[ L(X(n+1)) - L(X(n)) | X(n)\right]  %&\leq \sum_{i \in [N]}X_i(n)\lambda_i-\max\{X_i(n): i \in [N]\} + \frac{1}{2}(NC+1)\\
&\leq \max\{X_i(n): i \in [N]\}(\sum_{i \in [N]}\lambda_i - 1) +  \frac{1}{2}(NC+1).
}
%\begin{align*}
%&= \frac{1}{2} \sum_{i=1}^{N} \Bigg[ 2x_i \E \Big[ (A_i(t-1) - R_i(t-1)| X(t-1) = x \Big] + \E \Big[ (A_i(t-1) - R_i(t-1))^2 | X(t-1) = x \Big] \Bigg]\\
%&= \sum_{i=1}^{N}x_i \lambda _i - \sum_{i=1}^{N}x_i R_i(t-1) + \frac{N}{2}(1+C)\\
%&= \sum_{i=1}^{N}x_i \lambda _i + \frac{N}{2}(1+C) - \max _i x_i\\
%& \leq \frac{N}{2}(1+C) + (\max _i x_i)(\sum_{i=1}^{N} \lambda _i - 1)\\
%&= C_1 - \epsilon (\max _i x_i)
%\end{align*}
For a fixed $k$, any member $x$ of the level set $F(L,k)$ satisfies $\norm{x}_2^2 \leq 2k$. 
%\begin{xalignat*}{3}
%&\norm{x}_2^2 \leq 2k, && \max\{x_i: i \in [N]\} \leq \sqrt{2k}.
%\end{xalignat*} 
We can then bound the Lyapunov function at time $(n+1)$ conditioned on $X(n) \in F(L,k)$ by 
\eq{
\E[L(X(n+1))|X(n)] &\leq 2k + \frac{1}{2}(NC+1).
}
Further, when $x \notin F(L,k)$ then 
\eq{
\max\{x_i: i \in [N]\} &\geq \frac{\sqrt{2k}}{N}. 
}
Hence, we have the conditional drift for $X(n) \notin F(L,k)$
\eq{
\E\left[ L(X(n+1)) - L(X(n)) | X(n)\right]  &\leq -\frac{\sqrt{2k}}{N}\delta+  \frac{1}{2}(NC+1).
}
For some $\epsilon > 0$, we choose 
\begin{xalignat*}{3}
&k  > \frac{1}{2}\left(\frac{N\epsilon}{\delta} +  \frac{N}{2\delta}(NC+1)\right)^2,&&K > 2k + \frac{1}{2}(NC+1).
\end{xalignat*}
Foster's theorem applied to this choice of $k, K, \epsilon$ shows that states in $F(L,k)$ are positive recurrent. 
Since, the Markov chain is irreducible, the entire Markov chain is positive recurrent. 
\end{proof}
\end{document}