lecture-20.tex

% !TEX spellcheck = en_US
% !TEX spellcheck = LaTeX
\documentclass[a4paper,10pt,english]{article}
\input{header}
\title{Lecture 20: Reversible Processes and Queues}
\author{}
\begin{document}
\maketitle

\section{Examples of reversible processes}
%\begin{shaded*}
%\begin{exmp}[An Ergodic Random Walk] 
%Any ergodic, positive recurrent random walk is time reversible. 
%The transition probability satisfy the following equation for each $i \in \Z$
%\eq{
%P_{i,i+1}+P_{i,i-1}=1.
%} For every $n$ transitions from $i+1$ to $i$, there must be at least $n-1$ or at most $n+1$ transitions from $i$ to $i+1$. 
%The rate of transitions from $i+1$ to $i$ must hence be same as the number of transitions from $i$ to $i+1$. 
%It follows that, the process is time reversible.
%\end{exmp}
\subsection{Birth-death processes}
%\begin{exmp}[Birth-death processes]
%\label{exmp:birthdeath}
%\begin{defn}
We define two non-negative sequences birth and death rates denoted by $\{\lambda_n: n \in \N_0\}$ and $\{\mu_n: n \in \N_0\}$. 
A Markov process $\{X_t \in \N_0: t \in \R\}$ on the state space $\N_0$ is called a \textit{birth-death process} if its infinitesimal transition probabilities satisfy
\eq{
 P_{n,n+m}(h) = \begin{cases}% brackets may be (...), [...], \{...\}, or left out
      \lambda_n h + o(h), & \text{if } m = 1, \\
      \mu_n h + o(h), & \text{if } m = -1, \\
      o(h), & \text{if } |m| > 1.
   \end{cases}
}
%for some nonnegative numbers $\lambda_0, \lambda_1, \ldots$ (`birth rates') and $\mu_0, \mu_1, \ldots$ (`death rates'). %\\
%({\bf Note.} 
We say $f(h) = o(h)$ if $\lim_{h \to 0} f(h)/h = 0$.
%)
%\end{defn}
In other words, a birth-death process is any CTMC with generator of the form 
\eq{ 
   Q = \begin{pmatrix} % brackets may be (...), [...], \{...\}, or left out
      -\lambda_0 & \lambda_0 & 0 & 0 & 0\\
      \mu_1 & -(\lambda_1 + \mu_1) & \lambda_1 & 0 & 0 & \cdots\\
      0 & \mu_2 & -(\lambda_2 + \mu_2) & \lambda_2 & 0 & \cdots\\
      0 & 0 & \mu_3 & -(\lambda_3 + \mu_3) & \lambda_3 & \cdots \\
      \vdots & \vdots & \vdots & \vdots & \vdots  & \ddots
   \end{pmatrix}.
}
%\end{exmp}
%\end{shaded*}

\begin{prop}
An ergodic birth-death process in steady-state is time-reversible.
\end{prop}
\begin{proof}
%To prove the above, we must show that the rate at which the process goes from state $i$ to $i+1$ is equal to the rate of going from $i+1$ to $i$. 
%However, during any time interval of length $t$, the number of transitions from $i$ to $i+1$ should be within $1$ of the number of transitions from $i+1$ to $i$ (since the process is birth and death process. 
%Hence, as $t \to \infty$,  both rates will be equal. 
Since the process is stationary, the probability flux must balance across any cut of the form $A = \{0, 1, 2, \ldots, i\}$, $i \geq 0$. % (Lemma \ref{lem:fluxbalance}). 
But, this is precisely the equation $\pi_i \lambda_i = \pi_j \mu_j$ since there are no other transitions possible across the cut. 
So the process is time-reversible. 
\end{proof}

In fact, the following, more general, statement can be proven using similar ideas. 

\begin{prop}
Consider an ergodic CTMC on a countable state space $I$ with the following property: for any pair of states $i \neq j \in I$, there is a {\textit unique} path $i = i_0 \to i_1 \to \cdots \to i_{n(i,j)} = j$ of distinct states having positive probability. Then the CTMC in steady-state is reversible. 
\end{prop}

%\begin{shaded*}
%\begin{exmp}[Truncated CTMCs]
\subsection{Truncated Markov Processes}
%\begin{cor}
%Consider an M/M/s queue with Poisson$(\lambda)$ arrivals and each server having exponential service time $\exp(\mu)$ service. If $\lambda > s \mu$, then the output process in steady state is Poisson$(\lambda)$.
%\end{cor}
%\begin{proof}
%Let $X(t)$ denote the number of customers in the system at time $t$. Since M/M/s process is a birth and death process, it follows from the previous proposition that $\{X(t),~t \geq 0\}$ is time reversible. Now going forward in time, the time instants at which $X(t)$ increases by 1 are the arrival instants of a Poisson process. Hence, by time reversibility, the time Points at which $X(t)$ increases by 1 when we go backwards in time also constitutes a Poisson process. But these instants are exactly the departure instants of the forward process. Hence the result.
%\end{proof}
%\begin{defn}
Consider a transition rate matrix $(Q_{ij})_{i, j \in I}$ on the countable state space $I$. 
Given a nonempty subset $A \subseteq I$, the truncation of $Q$ to $A$ is the transition rate matrix $\{Q^A_{ij}: i, j \in A\}$, where for all $i, j \in A$
\eq{
 Q^A_{ij} = \begin{cases}
Q_{ij}, & j \neq i, \\
-\sum_{k \neq i, k \in A} Q_{ik}, &  j = i.
\end{cases}
}
%\end{defn}

\begin{prop}
Suppose $\{X_t: t \in \R\}$ is an irreducible, time-reversible CTMC on the countable state space $I$, 
with generator $Q = \{Q_{ij}: i, j \in I\}$ and stationary probabilities $\pi = \{\pi_j: j \in I\}$. 
Suppose the truncation $Q^A$ is irreducible for some $A \subseteq I$. 
Then, any stationary CTMC with state space $A$ and generator $Q^A$ is also time-reversible, with stationary probabilities
\begin{align*}
\pi_j^A&=\frac{\pi_j}{\sum_{i \in A}\pi_i},~ j \in A.
\end{align*}
\end{prop}
\begin{proof}
It is clear that $\pi^A$ is a distribution on state space $A$. 
We must show the reversibility with this distribution $\pi^A$. 
That is, we must show for all $i, j \in A$
\begin{align*}
\pi_i^AQ_{ij}&=\pi_j^AQ_{ji}.
\end{align*}
%However, this is equivalent to showing 
%\begin{align*} 
%\pi_iQ_{ij}&=\pi_jQ_{ji},~ i \in A,~ j \in A.
%\end{align*}
However, this is true since the original chain is time reversible.
\end{proof}

%\end{exmp}
%\end{shaded*}

%\begin{shaded*}

%\begin{exmp}[Random walks on edge-weighted graphs] 
\subsection{Random walks on edge-weighted graphs}
Consider an undirected graph $G = (I,E)$ with the vertex set $I$ and the edge set $E$ being a subset of unordered pairs of elements from $I$. 
Assume having a positive number $w_{ij}$ associated with each edge $\{i,j\}$ in $E$. 
Further the edge weight $w_{ij}$ is defined to be 0 if $\{i,j\}$ is not an edge of the graph. 
Suppose that a particle moves in discrete time, from one vertex to another in the following manner: If the particle is presently at vertex  $i$ then it will next move to vertex $j$ with probability
\eq{
P_{ij} &=\frac{w_{ij}}{\sum_{j}w_{ij}}.
}
%where $w_{ij}$ is defined to be 0 if $\{i,j\}$ is not an edge of the graph. 
The Markov chain describing the sequence of vertices visited by the particle is a random walk on an undirected edge-weighted graph. 
%\end{exmp}
%\begin{note}
Google's PageRank algorithm, to estimate the relative importance of webpages, is essentially a random walk on a graph!
%\end{note}

%\end{shaded*}

\begin{prop}
Consider an irreducible Markov chain that describes the random walk on an edge weighted graph with a finite number of vertices. 
%If this Markov chain is irreducible, then it is, in steady state,
In steady state, this Markov chain is time reversible with stationary probability of being in a state $i \in I$ given by 
\begin{align}
\label{eqn:graphrwstationarydist}
\pi_i &= \frac{\sum_{j}w_{ij}}{\sum_{j}\sum_{k}w_{kj}}. %, \quad \forall i \in I.
\end{align}
\end{prop}
\begin{proof}
Using the definition of transition probabilities for this Markov chain, 
we notice that the detailed balance equation for each pair of states $i, j \in I$ 
%\begin{equation*}
%\alpha_iP_{ij}=\alpha_{j}P_{ji}
%\end{equation*}
reduces to 
\eq{
 \frac{\alpha_i w_{ij}}{\sum_{k}w_{ik}} &=\frac{\alpha_j w_{ji}}{\sum_{k}w_{jk}}.
}
From the symmetry of edge weights in undirected graphs, it follows that $w_{ij}=w_{ji}$. 
Hence, we see that the distribution $\pi$ defined as in~\eqref{eqn:graphrwstationarydist} solves the equation, and we get the desired result.
\end{proof}

%\begin{shaded*}
%\begin{exmp}[Random walks on graphs]
%
%%\begin{defn}
%Consider a finite graph $G = (I, E)$, $|I| < \infty$, $E \subseteq I \times I$, with a nonnegative edge weight for each edge, $w: E \to \R_+$. 
%We can consider a (discrete time) random walk on this graph, with the state being the location of a single particle on one of the nodes of this graph. 
%For any node $i \in I$, we can define the set of incoming and outgoing edges as 
%\begin{xalignat*}{3}
%&H_{i} = \{(j,i) \in E: j \in I\},&&F_i = \{(i,j) \in E: j \in I\}.
%\end{xalignat*}
%The probability of the particle moving from node $i$ to node $j$ in a single unit of time is defined by 
%\begin{align*}
%P_{ij} = \begin{cases}
%\frac{w_{i,j}}{\sum_{f \in F_i}w_{f}}, & (i,j) \in E, \\
%0, & (i,j) \notin E.
%\end{cases}
%\end{align*}
%The Markov chain $\{X(n) \in I: n \in \N\}$, with the above transition probability matrix, describing the sequence of vertices visited by the particle is called a \textbf{random walk on an edge-weighted graph}. 
%%The associated transition matrix for each $i,j \in I$ is 
%%\eq{
%%P_{ij} &= P_e.
%%}
%%\end{defn}
%\end{exmp}
%
%\end{shaded*}

The following `dual' result also holds:

\begin{lem} 
Let $\{X\}_n$ be a reversible Markov chain on a finite state space $I$ and transition probability matrix $P$. 
Then, there exists a random walk on a weighted, undirected graph $G$ with the same transition probability matrix $P$. 
\end{lem}
\begin{proof} 
We create a graph $G = (I,E)$, where $(i,j) \in E$ if and only if $P_{ij} > 0$. We then set edge weights 
\begin{align*}
w_{ij} \triangleq \pi_i P_{ij} = \pi_jP_{ji} = w_{ji},
\end{align*}
where $\pi$ is the stationary distribution of $X$. With this choice of weights, it is easy to check that $w_i = \sum_j w_{ij} = \pi_i$, and the transition matrix associated with a random walk on this graph is exactly $P$.
\end{proof}


%%The sequence $\{X_n,X_{n-1} \hdots \}$ is called reverse process. Let $P^*$ denote the transition probability matrix. 
%%\begin{align*}
%%P_{ij}^*&=P(X_{n-1}=j|X_n=i)=\frac{P(X_{n-1}=j,X_n=i)}{P(X_n=i)}\\
%%&=\frac{P(X_{n-1}=j)P(X_n=i|X_{n-1}=j)}{P(X_{n}=i)}\\
%%&=\frac{P(X_{n-1}=j)}{P(X_{n}=i)}P_{ji}
%%\end{align*}
%%suppose we are considering a stationary Markov chain, $P(X_n=l)=P(X_{n-1}=l)=\alpha(l),~\forall l$, $\alpha(i){P^*}_{ij}=\alpha(j)(P)_{ji}$. 
%\begin{lem} A stationary Markov chain with transition matrix $P$ is reversible if the reversed process follows the same probabilistic law as the original process, i.e. $P^*= P$. Any non-negative vector $\alpha$ satisfying $\alpha_iP_{ij}=\alpha_jP_{ji},~\forall i,j \in I$ and $\sum_{j \in I}\alpha_j=1$ is stationary distribution of this Markov chain. 
%\end{lem}
%\begin{lem} A stationary Markov process with generator matrix $Q$ is reversible if the reversed process follows the same probabilistic law as the original process, i.e. $Q^*= Q$. Any non-negative vector $\pi$ satisfying $\pi_iQ_{ij}=\pi_jQ_{ji},~\forall i,j \in I$ and $\sum_{j \in I}\pi_j=1$ is stationary distribution of this Markov process. 
%\end{lem}
%\begin{proof} Let $\pi$ be a stationary distribution of  Since $Q^{\ast}_{ij} = \frac{\pi_j}{\pi_i}Q_{ji}$ and the condition for time reversibility is given by $\pi_i Q_{ij}=\pi_jQ_{ji}$. This is true because,
%
%It is clear that $\pi$ is an equilibrium distribution of forward Markov process, since
%\begin{align*}
%\sum_{i} \pi_i Q_{ij}=\sum_i \pi_j Q_{ji}=\pi_j \sum \alpha_i = 1.
%\end{align*}
%Since stationary probabilities are the unique solution of the above, the claim follows.
%\end{proof}
%\begin{proof} The condition for time reversibility is given by $\alpha_i P_{ij}=\alpha_jP_{ji}$. This is true because,
%\begin{align*}
%\sum_{i} \alpha_i P_{ij}=\sum_i \alpha_j P_{ji}=\alpha_j \sum \alpha_i = 1.
%\end{align*}
%Since stationary probabilities are the unique solution of the above, the claim follows.
%\end{proof}


%\subsection{Time Reversibility of Discrete Time Markov Chains}
%Consider a discrete time Markov chain with transition probability matrix $P$ and stationary probability vector $\alpha$.\\
%\textbf{Claim:} The reversed process is a Markov chain.
%\begin{proof}
%\begin{align*}
%&P(X_{m-1}=i|X_m=j,X_{m+1}=i_{m+1} \hdots )=\frac{P(X_{m-1}=i,X_m=j, \hdots )}{P(X_m=j, X_{m+1}=i_{m+1}\hdots )}\\
%&=\frac{P(X_{m-1}=i,X_m=j)P(X_{m+1}=i_{m+1}\hdots |X_{m-1}=i,X_m=j  )}{P(X_m=j)P(X_{m+1}=i_{m+1} \hdots |X_m=j)}\\
%&\stackrel{(a)}{=}\frac{P(X_{m-1}=i,X_m=j)P(X_{m+1}=i_{m+1}\hdots |X_m=j  )}{P(X_m=j)P(X_{m+1}=i_{m+1} \hdots |X_m=j)}\\
%&=P(X_{m-1}=i|X_m=j).\\
%\end{align*}
%where $(a)$ follows from the Markov property.
%\end{proof}
%The sequence $\{X_n,X_{n-1} \hdots \}$ is called reverse process. Let $P^*$ denote the transition probability matrix. 
%\begin{align*}
%P_{ij}^*&=P(X_{n-1}=j|X_n=i)=\frac{P(X_{n-1}=j,X_n=i)}{P(X_n=i)}\\
%&=\frac{P(X_{n-1}=j)P(X_n=i|X_{n-1}=j)}{P(X_{n}=i)}\\
%&=\frac{P(X_{n-1}=j)}{P(X_{n}=i)}P_{ji}
%\end{align*}
%suppose we are considering a stationary Markov chain, $P(X_n=l)=P(X_{n-1}=l)=\alpha(l),~\forall l$, $\alpha(i){P^*}_{ij}=\alpha(j)(P)_{ji}$. If $P^*=P^T$ then the Markov chain is called time reversible. Thus the condition for time reversibility is given by $\alpha P_{ij}=\alpha_jP_{ji}$. Any non-negative vector $X$ satisfying $X_iP_{ij}=X_jP_{ji},~\forall i,j$ and $\sum_{j \in \mathcal{N}_0}X_j=1$ is stationary distribution of time-reversible Markov chain. This is true because,
%\begin{align*}
%\sum_{i} X_i P_{ij}=\sum_i X_j P_{ji}=X_j \sum X_i = 1.
%\end{align*}
%Since stationary probabilities are the unique solution of the above, the claim follows.\\




%
%\subsection{Example 4.7(E): Example 4.3(C) revisited}
%Example 4.3(C) was with regard to the age of a renewal process. $X_n$ the forward process there was such that it increases in steps of 1 until it hits a value chosen by the inter arrival distribution. Hence the reverse process should be such that it decreases in steps of 1 until it hits 1 and then jumps to a state as chosen by the inter arrival distribution. Thus letting $\pi_i$ as the probability of inter arrival, it seems likely that  $P_{1i}*=\pi_i, ~ P_{i,i-1}=1,~ i > 1$. We have that $P_{i,1}=\frac{\pi_i}{\sum_{j \geq 1}\pi_j}=1-P_{i,i+1}, ~ i \geq 1$. For the reversed chain to be given as above, we would need 
%\begin{align*}
%&\alpha_i P_{ij}=\alpha_j P_{ji}^*\\
%&\alpha_i \frac{\pi_i}{\sum_j \pi_j}=\alpha_1 \pi_i\\
%&\alpha_i=\alpha_1 P(X \geq i)\\
%&1=\sum_i \alpha_i=\alpha_1 E[X]; \alpha_i=\frac{P(X \geq i)}{E[X]}, 
%\end{align*}
%where $X$ is the inter arrival time. We need to verify that $\alpha_i P_{i,i+1}=\alpha_{i+1}P^*_{i+1,i}$ or equivalently $P(X \geq i)(1-\frac{\pi_i}{P(X \geq i)})=P(X \geq i)$ to complete the proof that the reversed process is the excess process and the limiting distributions are as given above. But that is immediate.
%

%\subsection{Time Reversibility for CTMC}
%\begin{prop}
%Markov property holds going backward in time. i.e.
%\begin{equation*}
%Pr(X(t-s)=j|X(t)=i,X(y), y > t)=Pr(X(t-s)=j|X(t)=i).
%\end{equation*}
%\begin{proof}
%\begin{align*}
%&Pr(X(t-s)=j|X(t)=i, X(y), y>t)=\frac{Pr(X(t-s)=j, X(t)=i, X(y), y>t)}{Pr(X(t)=i, X(y), y>t)}\\
%&\stackrel{(a)}{=}\frac{Pr(X(t-s)=j)Pr(X(t)=i|X(t-s)=j)Pr(X(y), y>t|X(t)=i)}{Pr(X(t)=i, X(y), y>t)}\\
%&\stackrel{}{=}\frac{Pr(X(t-s)=j)Pr(X(t)=i|X(t-s)=j)}{Pr(X(t)=i)}\\
%&\stackrel{}{=}Pr(X(t-s)=j|X(t)=i).\\
%\end{align*}
%where $(a)$ follows from the Markov property of forward chain.  
%\end{proof}
%\end{prop}
%Now we would like to characterize the sojourn time in each state. Since the amount of time spent in each state is same whether in forward or backward direction, one expects the sojourn time for the reverse chain to be exponential with the same parameter $\nu_i$. We can verify this formally:
%\begin{align*}
%&Pr(\text{Process in state i during [t-s,t]}|X(t)=i)\\
%&= {Pr(\text{Process in state i during [t-s,t]})}/{Pr(X(t)=i)}\\
%&= \frac{Pr(X(t-s)=i)e^{-\nu_i s}}{Pr(X(t)=i)}=e^{-\nu_i s}.\\
%\end{align*}
%Next, we would like to characterize the transition probabilities of the reversed chain. The sequence of states visited by the reverse process constitutes a discrete-time Markov chain with transition probabilities $P_{ij}^*$ give by 
%\begin{equation}
%P_{ij}^*= \frac{\pi_jP_{ji}}{\pi_i},
%\end{equation} 
%where $\{\pi_i,~ i \geq 0\}$ are the stationary probabilities of the embeddede discrete time Markov chain with transition probabilities $P_{ij}$. Denote 
%\begin{equation}
%q_{ij}^*=\nu_iP_{ij}^*
%\end{equation}
%as the infinitesimal rates of the reversed process. Using the previous formula for $P_{ij}^*$ we see that
%\begin{equation}
%q_{ij}^*=\frac{\nu_i\pi_jP_{ji}}{\pi_i}.
%\end{equation} 
%Recall that 
%\begin{equation}
%P_{k}=\frac{\pi_k / \nu_k}{\sum_{i}\pi_i / \nu_i},
%\end{equation}
%we see that 
%\begin{equation}
%\frac{\pi_i}{\pi_j}=\frac{\nu_j P_j}{\nu_i P_i}.
%\end{equation}
%Hence, 
%\begin{equation}
%q_{ij}^*=\frac{\nu_j P_{j}P_{ji}}{P_i}= \frac{P_jq_{ji}}{P_i}.
%\end{equation}
%That is,
%\begin{equation*}
%P_{i}q_{ij}^*=P_{j}q_{ji}.
%\end{equation*}
%Form the above equation, it is obvious that 
%\begin{equation*}
%\sum_{i}P_i q_{ij}^*=0.
%\end{equation*} 
%This implies that $P_j,~j \geq 0$ are the stationary probabilities for the reverse chain as $P_j$s satisfy the balance equations.
%%\begin{defn}
%A stationary CTMC is said to be time reversible if the reverse process follows the same probabilistic law as the original process. That is, if, for all $i,j$
%\begin{equation*}
%q_{ij}^*= q_{ij} \iff P_{i}q_{ij}=P_jq_{ji}.
%\end{equation*} 
%%\end{defn} 

\section{General queueing theory}
%\begin{note}
The notation A/S/N/B/D for a queueing system indicates 
\begin{itemize}
\item[A:] Inter-arrival time distribution, 
\item[S:] Service time distribution, 
\item[N:] Number of servers, 
\item[B:] Maximum number of jobs that can be waiting and in service at any time ($\infty$ by default), and 
\item[D:] Queueing service discipline (FIFO by default). 
\end{itemize}
%\end{note}

%%% IMPROVE PASTA PROOF
\begin{thm}[PASTA] Poisson arrivals see time averages. 
At any time $t$, we denote a system state by $N(t)$ and the number of arrivals in $[0,t)$ by $A(t)$. 
The $n$th arrival instant is denoted by $A_n$ and $B$ a Borel measurable set in $\R_+$, then 
\eq{
\bar{\tau}_B &\triangleq \lim_{t \in \R_+} \frac{1}{t}\int_0^{t}1\{N(u) \in B\}du =  \lim_{n \in \N} \frac{1}{n}\sum_{i=1}^{n}1\{N(A_i-)\in B \} \triangleq \bar{c}_B.
}
\end{thm}
\begin{proof}
We will show the special case when $N(t)$ is the number of customers in the system at time $t$, and $B = \{k\}$. 
We define for $k \in \N_0$
\begin{xalignat*}{3}
&P_k \triangleq \lim_{t\in\R_+}\Pr\{N(t) = k\}&&A_k \triangleq \lim_{t \in \R_+}\Pr\{N(t-) = k| A_{k+1} = t\}.
\end{xalignat*}
Using independent increment property of Poisson arrivals, Baye's rule, and continuity of probabilities, we can write the second limiting probability as 
\eq{
A_k &= \lim_{t \in \R_+}\lim{h \downarrow 0}\frac{\Pr\{N(t-)=k,A(t+h)-A(t) = 1\}}{\Pr\{A(t+h)-A(t) = 1\}} = \lim_{t \in \R_+}\Pr\{N(t) = k\} = P_k.
}
\end{proof}
\begin{thm}[Little's law] Consider a stable single server queue. 
Let $T_i$ be waiting time of customer $i$, $N(t)$ be the number of customers in the system at time $t$, and $A(t)$ be the number of customers that entered system in duration $[0,t)$, then
\begin{align*}
\lim_{t \to \infty}\frac{\int_{0}^tN(u)du}{t} &= \lim_{t \to \infty} \frac{\sum_{i=1}^{A(t)}T_i}{A(t)}.
\end{align*} 
\end{thm}
\begin{proof}
Let $A(t), D(t)$ respectively denote the number of arrivals and departures in time $[0,t)$. Then, we have 
\begin{align*}
\sum_{i=1}^{D(t)}T_i \leq \int_{0}^tN(u)du \leq \sum_{i=1}^{A(t)}T_i.
\end{align*}
Further, for a stable queue we have
\begin{align*}
\lim_{t \to \infty}\frac{D(t)}{t} = \lim_{t \to \infty}\frac{A(t)}{t}.
\end{align*}
Combining these two results, the theorem follows.
\end{proof}

%\begin{shaded*}
%\begin{exmp}[The M/M/1 queue]
\subsection{The M/M/1 queue}
The M/M/1 queue is the simplest and most studied models of queueing systems. 
We assume a continuous-time queueing model with following components. 
\begin{itemize}
\item There is a single queue for waiting that can accommodate arbitrarily large number of customers.  
\item Arrivals to the queue occur according to a Poisson process with rate $\lambda > 0$. 
That is, let $A_n$ be the arrival instant of the $n$th customer, then the sequence of inter-arrival times $\{A_{n}-A_{n-1} : n \in \N\}$ is \textit{iid} exponentially distributed with rate $\lambda$. 
\item There is a single server and the service time of $n$th customer is denoted by a random variable $S_n$. 
The sequence of service times $\{S_n: n \in \N\}$ are \textit{iid} exponentially distributed with rate $\mu > 0$, 
independent of the Poisson arrival process. 
\item We assume that customers join the tail of the queue, and hence begin service in the order that they arrive \textit{first-in-queue-first-out} (FIFO).
\end{itemize}
Let $X(t)$ denote the number of customers in the system at time $t \in \R_+$, 
where ``system" means the queue plus the service area. 
For example, $X(t) = 2$ means that there is one customer in service and one waiting in line. 
Due to continuous distributions of inter-arrival and service times, a transition can only occur at customer arrival or departure times. 
Further, departures occur whenever a service completion occurs. 
Let $D_n$ denote the $n$th departure from the system. 
At an arrival time $A_n$,  the number $X(A_n) = X(A_n-)+1$ jumps up by the amount 1, 
whereas at a departure time $D_n$, then number $X(D_n) = X(D_n-)-1$ jumps down by the amount 1. 
%(courtesy Karl Sigman's lecture notes, \url{http://www.columbia.edu/~ks20/stochastic-I/stochastic-I-CTMC.pdf})


%\end{exmp}
%\end{shaded*}



For the M/M/1 queue, one can argue that $\{X(t): t \in \R_+\}$ is a CTMC on the state space $\N_0$. 
We will soon see that a \textit{stable} M/M/1 queue is time-reversible.   
%\begin{itemize}
%\item
\subsubsection{Transition rates} 
Given the current state $\{X(t) = i\}$, the only transitions possible in an infinitesimal time interval are (a) a single customer arrives, or (b) a single customer leaves (if $i \geq 1$). 
It follows that the infinitesimal generator for the CTMC $\{X(t)\}_t$ is 
\eq{
Q_{ij} &= \begin{cases}
\lambda, &j = i+1,\\
\mu, &j = i-1, \\
0, &|j-i| > 1. 
\end{cases}
}
Since $\lambda, \mu > 0$, this defines an irreducible CTMC. 

\subsubsection{Equilibrium distribution and reversibility} 
The M/M/1 queue's generator defines a birth-death process.  %(Example \ref{exmp:birthdeath}), 
Hence, if it is stationary, then it must be time-reversible, with the equilibrium distribution $\pi$ satisfying the detailed balance for each $i \in \N_0$
\eq{
\pi_i \lambda &= \pi_{i+1} \mu.
} 
This yields $\pi_{i+1} = \frac{\lambda}{\mu} \pi_i$. 
Since $\sum_{i \geq 0} \pi = 1$, we must have $\rho \triangleq \frac{\lambda}{\mu} < 1$, giving for each $i \in \N_0$
\eq{\pi_i = (1-\rho) \rho^i.
} 
In other words, if $\lambda < \mu$, then the equilibrium distribution of the number of customers in the system is geometric with parameter $\rho = \lambda/\mu$. 
We say that the M/M/1 queue is in the \textit{stable} regime when $\rho < 1$. 
We have thus shown 
\begin{cor} 
The number of customers in an M/M/1 queueing system at equilibrium is a reversible Markov process.
\end{cor}
Further, since M/M/1 queue is a reversible CTMC, the following theorem follows. 
\begin{thm}[Burke] 
Departures from a stable M/M/1 queue are Poisson with same rate as the arrivals.
\end{thm}

\subsubsection{Limiting waiting room: M/M/1/$K$} 
Consider a variant of the M/M/1 queueing system that has a finite buffer capacity of at most $k$ customers. Thus, customers that arrive when there are already $k$ customers present are `rejected'. It follows that the CTMC for this system is simply the M/M/1 CTMC truncated to the state space $\{0, 1, \ldots, K\}$, and so it must be time-reversible with stationary distribution $\pi_i = \rho^i / \sum_{j = 0}^k \rho^j$, $0 \leq i \leq k$. 
%\end{itemize}




%\begin{cor}
%Consider an M/M/s queue with Poisson$(\lambda)$ arrivals and each server having exponential service time $\exp(\mu)$ service. If $\lambda > s \mu$, then the output process in steady state is Poisson$(\lambda)$.
%\end{cor}
%\begin{proof}
%Let $X(t)$ denote the number of customers in the system at time $t$. Since M/M/s process is a birth and death process, it follows from the previous proposition that $\{X(t),~t \geq 0\}$ is time reversible. Now going forward in time, the time instants at which $X(t)$ increases by 1 are the arrival instants of a Poisson process. Hence, by time reversibility, the time Points at which $X(t)$ increases by 1 when we go backwards in time also constitutes a Poisson process. But these instants are exactly the departure instants of the forward process. Hence the result.
%\end{proof}

\begin{shaded*}
\begin{exmp}[Two queues with joint waiting room] Consider two independent M/M/1 queues with arrival and service rates $\lambda_i$ and $\mu_i$ respectively for $i \in [2]$. Then, joint distribution of two queues is
\begin{align*}
\pi(n_1,n_2) &= (1 -\rho_1)\rho_1^{n_1}(1-\rho_2)\rho_2^{n_2},~~n_1,n_2 \in \N_0. 
\end{align*}
Suppose both the queues are sharing a common waiting room, where if arriving customer finds $R$ waiting customer then it leaves. In this case,
\begin{align*}
\pi(n_1,n_2) &= (1 -\rho_1)\rho_1^{n_1}(1-\rho_2)\rho_2^{n_2},~~(n_1,n_2) \in A \subseteq \N_0\times\N_0. 
\end{align*}
\end{exmp}
\end{shaded*}

\end{document}

\section{Reversed Processes}
\begin{defn} Let $X(t)$ be a stochastic process then $X(\tau-t)$ is the reversed process.
\end{defn}
\begin{lem} If $X(t)$ is a time homogeneous non-stationary Markov chain then the reversed process $X(\tau -t)$ is a non time-homogenous Markov chain.
\end{lem}
\begin{proof} Let $\F_m = \cup_{k \geq m}\{X_k = i_k\}$. Then, we can write
\begin{align*}
\Pr\{X_{m-1}=i|X_m=j,\F_{m+1}\} %&= \frac{\Pr\{X_{m-1}=i,X_m=j, \F_{m+1} \}}{\Pr\{X_m=j, \F_{m+1}\}}\\
&=\frac{\Pr\{X_{m-1}=i|X_m=j\}\Pr\{\F_{m+1} |X_{m-1}=i,X_m=j \}}{\Pr\{\F_{m+1} |X_m=j\}}.
\end{align*}
Result follows from Markov property of $X(t)$, i.e.
\begin{align*}
Pr\{\F_{m+1} |X_m= j, X_{m-1} = i \} &=\Pr\{\F_{m+1}=i|X_m=j\}.
\end{align*}
\end{proof}
%\begin{lem}  If $X(t)$ is a stationary Markov chain with transition matrix $P$ and equilibrium distribution $\alpha$, then the reversed chain $X(\tau -t)$ is a stationary Markov chain with same equilibrium distribution $\alpha$ and transition matrix $P^{\ast}$ such that
%\begin{align*}
%P^{\ast}_{ij} = \frac{\alpha_j}{\alpha_i}P_{ji}.
%\end{align*}
%\end{lem}
%\begin{proof} Easy to verify from definition of reversibility that 
%\begin{align*}
%\Pr\{X(t+1) = j, X(t) = i\} = \Pr\{X(t+1) = i, X(t) = j\}.
%\end{align*}
%Also, it's easy to check that $\alpha P^{\ast} = \alpha$.
%\end{proof}

\begin{lem} If $X(t)$ is a stationary Markov process with generator matrix $Q$ and equilibrium distribution $\pi$, then the reversed process $X(\tau -t)$ is a stationary Markov process with same equilibrium distribution $\pi$ and generator matrix $Q^{\ast}$ such that
\begin{align*}
Q^{\ast}_{ij} = \frac{\pi_j}{\pi_i}Q_{ji}.
\end{align*}
\end{lem}
\begin{proof} Easy to verify from definition of reversibility that 
\begin{align*}
\Pr\{X(t+h) = j, X(t) = i\} = \Pr\{X(t+h) = i, X(t) = j\}.
\end{align*}
Also, it's easy to check that $\pi Q^{\ast} = 0$.
\end{proof}

%%The sequence $\{X_n,X_{n-1} \hdots \}$ is called reverse process. Let $P^*$ denote the transition probability matrix. 
%%\begin{flalign*}
%%P_{ij}^*&=P(X_{n-1}=j|X_n=i)=\frac{P(X_{n-1}=j,X_n=i)}{P(X_n=i)}\\
%%&=\frac{P(X_{n-1}=j)P(X_n=i|X_{n-1}=j)}{P(X_{n}=i)}\\
%%&=\frac{P(X_{n-1}=j)}{P(X_{n}=i)}P_{ji}
%%\end{flalign*}
%%suppose we are considering a stationary Markov chain, $P(X_n=l)=P(X_{n-1}=l)=\alpha(l),~\forall l$, $\alpha(i){P^*}_{ij}=\alpha(j)(P)_{ji}$. 
%\begin{lem} A stationary Markov chain with transition matrix $P$ is reversible if the reversed process follows the same probabilistic law as the original process, i.e. $P^*= P$. Any non-negative vector $\alpha$ satisfying $\alpha_iP_{ij}=\alpha_jP_{ji},~\forall i,j \in I$ and $\sum_{j \in I}\alpha_j=1$ is stationary distribution of this Markov chain. 
%\end{lem}
\begin{lem} A stationary Markov process with generator matrix $Q$ is reversible if the reversed process follows the same probabilistic law as the original process, i.e. $Q^*= Q$. Any non-negative vector $\pi$ satisfying $\pi_iQ_{ij}=\pi_jQ_{ji},~\forall i,j \in I$ and $\sum_{j \in I}\pi_j=1$ is stationary distribution of this Markov process. 
\end{lem}
%\begin{proof} Let $\pi$ be a stationary distribution of  Since $Q^{\ast}_{ij} = \frac{\pi_j}{\pi_i}Q_{ji}$ and the condition for time reversibility is given by $\pi_i Q_{ij}=\pi_jQ_{ji}$. This is true because,
%
%It is clear that $\pi$ is an equilibrium distribution of forward Markov process, since
%\begin{align*}
%\sum_{i} \pi_i Q_{ij}=\sum_i \pi_j Q_{ji}=\pi_j \sum \alpha_i = 1.
%\end{align*}
%Since stationary probabilities are the unique solution of the above, the claim follows.
%\end{proof}
%\begin{proof} The condition for time reversibility is given by $\alpha_i P_{ij}=\alpha_jP_{ji}$. This is true because,
%\begin{align*}
%\sum_{i} \alpha_i P_{ij}=\sum_i \alpha_j P_{ji}=\alpha_j \sum \alpha_i = 1.
%\end{align*}
%Since stationary probabilities are the unique solution of the above, the claim follows.
%\end{proof}


%\subsection{Time Reversibility of Discrete Time Markov Chains}
%Consider a discrete time Markov chain with transition probability matrix $P$ and stationary probability vector $\alpha$.\\
%\textbf{Claim:} The reversed process is a Markov chain.
%\begin{proof}
%\begin{flalign*}
%&P(X_{m-1}=i|X_m=j,X_{m+1}=i_{m+1} \hdots )=\frac{P(X_{m-1}=i,X_m=j, \hdots )}{P(X_m=j, X_{m+1}=i_{m+1}\hdots )}\\
%&=\frac{P(X_{m-1}=i,X_m=j)P(X_{m+1}=i_{m+1}\hdots |X_{m-1}=i,X_m=j  )}{P(X_m=j)P(X_{m+1}=i_{m+1} \hdots |X_m=j)}\\
%&\stackrel{(a)}{=}\frac{P(X_{m-1}=i,X_m=j)P(X_{m+1}=i_{m+1}\hdots |X_m=j  )}{P(X_m=j)P(X_{m+1}=i_{m+1} \hdots |X_m=j)}\\
%&=P(X_{m-1}=i|X_m=j).\\
%\end{flalign*}
%where $(a)$ follows from the Markov property.
%\end{proof}
%The sequence $\{X_n,X_{n-1} \hdots \}$ is called reverse process. Let $P^*$ denote the transition probability matrix. 
%\begin{flalign*}
%P_{ij}^*&=P(X_{n-1}=j|X_n=i)=\frac{P(X_{n-1}=j,X_n=i)}{P(X_n=i)}\\
%&=\frac{P(X_{n-1}=j)P(X_n=i|X_{n-1}=j)}{P(X_{n}=i)}\\
%&=\frac{P(X_{n-1}=j)}{P(X_{n}=i)}P_{ji}
%\end{flalign*}
%suppose we are considering a stationary Markov chain, $P(X_n=l)=P(X_{n-1}=l)=\alpha(l),~\forall l$, $\alpha(i){P^*}_{ij}=\alpha(j)(P)_{ji}$. If $P^*=P^T$ then the Markov chain is called time reversible. Thus the condition for time reversibility is given by $\alpha P_{ij}=\alpha_jP_{ji}$. Any non-negative vector $X$ satisfying $X_iP_{ij}=X_jP_{ji},~\forall i,j$ and $\sum_{j \in \mathcal{N}_0}X_j=1$ is stationary distribution of time-reversible Markov chain. This is true because,
%\begin{flalign*}
%\sum_{i} X_i P_{ij}=\sum_i X_j P_{ji}=X_j \sum X_i = 1.
%\end{flalign*}
%Since stationary probabilities are the unique solution of the above, the claim follows.\\



\subsection{Age and excess times}
Consider a discrete time renewal process 
Example 4.3(C) was with regard to the age of a renewal process. 
Let $X_n$ be the forward process there was such that it increases in steps of 1 until it hits a value chosen by the inter arrival distribution. Hence the reverse process should be such that it decreases in steps of 1 until it hits 1 and then jumps to a state as chosen by the inter arrival distribution. Thus letting $\pi_i$ as the probability of inter arrival, it seems likely that  $P_{1i}*=\pi_i, ~ P_{i,i-1}=1,~ i > 1$. We have that $P_{i,1}=\frac{\pi_i}{\sum_{j \geq 1}\pi_j}=1-P_{i,i+1}, ~ i \geq 1$. For the reversed chain to be given as above, we would need 
\begin{flalign*}
&\alpha_i P_{ij}=\alpha_j P_{ji}^*\\
&\alpha_i \frac{\pi_i}{\sum_j \pi_j}=\alpha_1 \pi_i\\
&\alpha_i=\alpha_1 P(X \geq i)\\
&1=\sum_i \alpha_i=\alpha_1 E[X]; \alpha_i=\frac{P(X \geq i)}{E[X]}, 
\end{flalign*}
where $X$ is the inter arrival time. We need to verify that $\alpha_i P_{i,i+1}=\alpha_{i+1}P^*_{i+1,i}$ or equivalently $P(X \geq i)(1-\frac{\pi_i}{P(X \geq i)})=P(X \geq i)$ to complete the proof that the reversed process is the excess process and the limiting distributions are as given above. But that is immediate.


%\subsection{Time Reversibility for CTMC}
%\begin{prop}
%Markov property holds going backward in time. i.e.
%\begin{equation*}
%Pr(X(t-s)=j|X(t)=i,X(y), y > t)=Pr(X(t-s)=j|X(t)=i).
%\end{equation*}
%\begin{proof}
%\begin{flalign*}
%&Pr(X(t-s)=j|X(t)=i, X(y), y>t)=\frac{Pr(X(t-s)=j, X(t)=i, X(y), y>t)}{Pr(X(t)=i, X(y), y>t)}\\
%&\stackrel{(a)}{=}\frac{Pr(X(t-s)=j)Pr(X(t)=i|X(t-s)=j)Pr(X(y), y>t|X(t)=i)}{Pr(X(t)=i, X(y), y>t)}\\
%&\stackrel{}{=}\frac{Pr(X(t-s)=j)Pr(X(t)=i|X(t-s)=j)}{Pr(X(t)=i)}\\
%&\stackrel{}{=}Pr(X(t-s)=j|X(t)=i).\\
%\end{flalign*}
%where $(a)$ follows from the Markov property of forward chain.  
%\end{proof}
%\end{prop}
%Now we would like to characterize the sojourn time in each state. Since the amount of time spent in each state is same whether in forward or backward direction, one expects the sojourn time for the reverse chain to be exponential with the same parameter $\nu_i$. We can verify this formally:
%\begin{flalign*}
%&Pr(\text{Process in state i during [t-s,t]}|X(t)=i)\\
%&= {Pr(\text{Process in state i during [t-s,t]})}/{Pr(X(t)=i)}\\
%&= \frac{Pr(X(t-s)=i)e^{-\nu_i s}}{Pr(X(t)=i)}=e^{-\nu_i s}.\\
%\end{flalign*}
%Next, we would like to characterize the transition probabilities of the reversed chain. The sequence of states visited by the reverse process constitutes a discrete-time Markov chain with transition probabilities $P_{ij}^*$ give by 
%\begin{equation}
%P_{ij}^*= \frac{\pi_jP_{ji}}{\pi_i},
%\end{equation} 
%where $\{\pi_i,~ i \geq 0\}$ are the stationary probabilities of the embeddede discrete time Markov chain with transition probabilities $P_{ij}$. Denote 
%\begin{equation}
%q_{ij}^*=\nu_iP_{ij}^*
%\end{equation}
%as the infinitesimal rates of the reversed process. Using the previous formula for $P_{ij}^*$ we see that
%\begin{equation}
%q_{ij}^*=\frac{\nu_i\pi_jP_{ji}}{\pi_i}.
%\end{equation} 
%Recall that 
%\begin{equation}
%P_{k}=\frac{\pi_k / \nu_k}{\sum_{i}\pi_i / \nu_i},
%\end{equation}
%we see that 
%\begin{equation}
%\frac{\pi_i}{\pi_j}=\frac{\nu_j P_j}{\nu_i P_i}.
%\end{equation}
%Hence, 
%\begin{equation}
%q_{ij}^*=\frac{\nu_j P_{j}P_{ji}}{P_i}= \frac{P_jq_{ji}}{P_i}.
%\end{equation}
%That is,
%\begin{equation*}
%P_{i}q_{ij}^*=P_{j}q_{ji}.
%\end{equation*}
%Form the above equation, it is obvious that 
%\begin{equation*}
%\sum_{i}P_i q_{ij}^*=0.
%\end{equation*} 
%This implies that $P_j,~j \geq 0$ are the stationary probabilities for the reverse chain as $P_j$s satisfy the balance equations.
%\begin{defn}
%A stationary CTMC is said to be time reversible if the reverse process follows the same probabilistic law as the original process. That is, if, for all $i,j$
%\begin{equation*}
%q_{ij}^*= q_{ij} \iff P_{i}q_{ij}=P_jq_{ji}.
%\end{equation*} 
%\end{defn} 


\subsection{Simple Queues}
\begin{cor} Number of customers in a simple M/M/1 queue at equilibrium is a reversible Markov process.
\end{cor}

\begin{thm}[PASTA] Poisson arrivals see time averages.
\end{thm}
\begin{thm}[Little's law] Consider a stable single server queue. Let $T_i$ be waiting time of customer $i$, $N(t)$ be the number of customers in the system at time $t$, and $A(t)$ be the number of customers that entered system in duration $[0,t)$, then
\begin{align*}
\lim_{t \to \infty}\frac{\int_{0}^tN(u)du}{t} &= \lim_{t \to \infty} \frac{\sum_{i=1}^{A(t)}T_i}{A(t)}.
\end{align*} 
\end{thm}
\begin{proof}
Let $A(t), D(t)$ respectively denote the number of arrivals and departures in time $[0,t)$. Then, we have 
\begin{align*}
\sum_{i=1}^{D(t)}T_i \leq \int_{0}^tN(u)du \leq \sum_{i=1}^{A(t)}T_i.
\end{align*}
Further, for a stable queue we have
\begin{align*}
\lim_{t \to \infty}\frac{D(t)}{t} = \lim_{t \to \infty}\frac{A(t)}{t}.
\end{align*}
Combining these two results, the theorem follows.
\end{proof}

\subsection{Truncated Reversible Processes}

%\begin{cor}
%Consider an M/M/s queue with Poisson$(\lambda)$ arrivals and each server having exponential service time $\exp(\mu)$ service. If $\lambda > s \mu$, then the output process in steady state is Poisson$(\lambda)$.
%\end{cor}
%\begin{proof}
%Let $X(t)$ denote the number of customers in the system at time $t$. Since M/M/s process is a birth and death process, it follows from the previous proposition that $\{X(t),~t \geq 0\}$ is time reversible. Now going forward in time, the time instants at which $X(t)$ increases by 1 are the arrival instants of a Poisson process. Hence, by time reversibility, the time Points at which $X(t)$ increases by 1 when we go backwards in time also constitutes a Poisson process. But these instants are exactly the departure instants of the forward process. Hence the result.
%\end{proof}
\begin{prop}
A time-reversible chain with limiting probabilities $\pi_j,~ j \in I$, that is truncated to the set $A\subseteq I$ and remains irreducible is also time reversible and has limiting probabilities 
\begin{align*}
\pi_j^A&=\frac{\pi_j}{\sum_{i \in A}\pi_i},~ j \in A.
\end{align*}
\end{prop}
\begin{proof}
We must show that 
\begin{align*}
\pi_i^AQ_{ij}&=\pi_j^AQ_{ji},~ i \in A,~ j \in A,
\end{align*}
or equivalently,
\begin{align*} 
\pi_iQ_{ij}&=\pi_jQ_{ji},~ i \in A,~ j \in A.
\end{align*}
But this is true as the original chain is time reversible.
\end{proof}
\begin{exmp}[Two queues with joint waiting room] Consider two independent M/M/1 queues with arrival and service rates $\lambda_i$ and $\mu_i$ respectively for $i \in [2]$. Then, joint distribution of two queues is
\begin{align*}
\pi(n_1,n_2) &= (1 -\rho_1)\rho_1^{n_1}(1-\rho_2)\rho_2^{n_2},~~n_1,n_2 \in \N_0. 
\end{align*}
Suppose both the queues are sharing a common waiting room, where if arriving customer finds $R$ waiting customer then it leaves. In this case,
\begin{align*}
\pi(n_1,n_2) &= (1 -\rho_1)\rho_1^{n_1}(1-\rho_2)\rho_2^{n_2},~~(n_1,n_2) \in A \subseteq \N_0\times\N_0. 
\end{align*}
\end{exmp}